mirror of
https://github.com/pjh456/OxideJS.git
synced 2026-10-23 18:21:13 +08:00
deps/vm: vendor regress 0.11.1 补 Script/Script_Extensions 的 Unknown(Zzzz) 取值——上游表缺未分配码点集取值，规范合法模式 \p{Script=Unknown} 编译期 SyntaxError；fork 纯增量（0 行改动上游行，仅补区间表与解析臂），回归钉四则（未分配匹配/已分配不匹配/sc·scx·Zzzz 别名/既有取值不扰动）
This commit is contained in:
Vendored
+1
@@ -0,0 +1 @@
|
||||
{"v":1}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"git": {
|
||||
"sha1": "7e64ad5e6807b5503e5cc97a79e0f129b23c556b",
|
||||
"dirty": true
|
||||
},
|
||||
"path_in_vcs": ""
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
---
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
- package-ecosystem: "cargo"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
@@ -0,0 +1,31 @@
|
||||
name: autofix.ci
|
||||
on:
|
||||
workflow_call:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
autofix:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: autofix-${{ hashFiles('**/Cargo.lock') }}
|
||||
|
||||
- run: rustup toolchain install ${{ env.rust_clippy }} --profile minimal --component rustfmt --component clippy
|
||||
- run: rustup default ${{ env.rust_clippy }}
|
||||
|
||||
- run: cargo clippy --fix --workspace
|
||||
- run: cargo fmt --all
|
||||
|
||||
- uses: autofix-ci/action@7a166d7532b277f34e16238930461bf77f9d7ed8
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- "**"
|
||||
push:
|
||||
branches:
|
||||
- "**"
|
||||
|
||||
name: Continuous integration
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo check --verbose
|
||||
|
||||
test:
|
||||
name: Test Suite
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
features:
|
||||
- ""
|
||||
- "index-positions"
|
||||
- "prohibit-unsafe"
|
||||
- "utf16"
|
||||
- "index-positions,prohibit-unsafe"
|
||||
- "index-positions,utf16"
|
||||
- "prohibit-unsafe,utf16"
|
||||
- "index-positions,prohibit-unsafe,utf16"
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo test --features "${{ matrix.features }}" --verbose
|
||||
|
||||
clippy:
|
||||
name: Clippy
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo clippy --all-targets --features "backend-pikevm,std,index-positions,prohibit-unsafe,utf16" -- -D warnings
|
||||
|
||||
test-nightly:
|
||||
name: Test (Nightly)
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo test --all-features --verbose
|
||||
|
||||
doc:
|
||||
name: Documentation
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo doc --verbose --document-private-items
|
||||
env:
|
||||
RUSTDOCFLAGS: -D warnings
|
||||
@@ -0,0 +1,20 @@
|
||||
/target
|
||||
**/*.rs.bk
|
||||
Cargo.lock
|
||||
.vscode
|
||||
.vscode/*
|
||||
.DS_Store
|
||||
regexp-pcre.js
|
||||
|
||||
.claude/
|
||||
CLAUDE.md
|
||||
|
||||
# Unicode Database Files
|
||||
DerivedCoreProperties.txt
|
||||
DerivedBinaryProperties.txt
|
||||
DerivedGeneralCategory.txt
|
||||
DerivedNormalizationProps.txt
|
||||
CaseFolding.txt
|
||||
emoji-data.txt
|
||||
PropList.txt
|
||||
Scripts.txt
|
||||
Vendored
+34
@@ -0,0 +1,34 @@
|
||||
[package]
|
||||
name = "regress"
|
||||
version = "0.11.1"
|
||||
authors = ["ridiculousfish <corydoras@ridiculousfish.com>"]
|
||||
description = "A regular expression engine targeting EcmaScript syntax"
|
||||
license = "MIT OR Apache-2.0"
|
||||
repository = "https://github.com/ridiculousfish/regress"
|
||||
keywords = ["regex", "regexp"]
|
||||
edition = "2024"
|
||||
readme = "README.md"
|
||||
|
||||
[features]
|
||||
default = ["backend-pikevm", "std"]
|
||||
|
||||
std = ["memchr/std"]
|
||||
|
||||
# Enables the PikeVM backend.
|
||||
backend-pikevm = []
|
||||
|
||||
# Prefers indexes to pointers for bytecode IP and string positions, for the paranoid.
|
||||
index-positions = []
|
||||
|
||||
# Prohibits all uses of unsafe code, for the paranoid.
|
||||
prohibit-unsafe = []
|
||||
|
||||
# Enables UTF-16 support. This disables some optimizations, so it should only be used when necessary.
|
||||
utf16 = []
|
||||
|
||||
# Enables Pattern trait implementation for str::find, str::contains, etc.
|
||||
pattern = []
|
||||
|
||||
[dependencies]
|
||||
hashbrown = "0.16.0"
|
||||
memchr = { version = "2.4.0", default-features = false }
|
||||
Vendored
+201
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2020 ridiculous_fish
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2020 ridiculous_fish
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
Vendored
+71
@@ -0,0 +1,71 @@
|
||||
# regress - REGex in Rust with EcmaScript Syntax
|
||||
|
||||
oh no why
|
||||
|
||||
## Introduction
|
||||
|
||||
regress is a backtracking regular expression engine implemented in Rust, which targets JavaScript regular expression syntax. See [the crate documentation](https://docs.rs/regress) for more.
|
||||
|
||||
It's fast, Unicode-aware, has few dependencies, and has a big test suite. It makes fewer guarantees than the `regex` crate but it enables more syntactic features, such as backreferences and lookaround assertions.
|
||||
|
||||
## Syntax
|
||||
|
||||
regress targets the EcmaScript 2018 standard regexp syntax, including support for gnarly cases such as variable-width lookbehind assertions containing capture groups. Note that subsequent standard have mostly left regexp syntax untouched, with a few exceptions such as the 'v' flag, which is supported.
|
||||
|
||||
### That darn 'u' flag
|
||||
|
||||
You will be sad to learn that JavaScript does not use UTF-8.
|
||||
|
||||
Originally JavaScript was designed for UCS-2, with 16-bit characters. Later UCS-2 was supplanted with UTF-16; however this was not automatic for regexps, but instead required opt-in for each regexp, with the 'u' flag. For example, the grinning face emoji U+1F600 is represented in a JavaScript string as a surrogate pair U+D83D and U+DE00. In a regexp without the 'u' flag, these surrogate pairs are matched as distinct characters:
|
||||
|
||||
const s = "😀";
|
||||
const re = /./;
|
||||
const m = s.match(re);
|
||||
console.log(m); // returns \uD83D, high surrogate
|
||||
|
||||
This behavior is almost never desired but is required by the ES spec. It's also super-awkward to express in Rust, which uses UTF-8 extensively. See below for how regress handles this.
|
||||
|
||||
The 'u' flag doesn't just modify character sets; it _also_ affects other behaviors such as how case-insensitive matching works, and (most bizarrely) the behavior of backreferences like \2. For example, in non-Unicode mode, \2 is a backreference if there are at least two capture groups; otherwise it is an octal escape (!).
|
||||
|
||||
regress mostly ignores the 'u' flag for character decoding - that's instead given by the call site (see below). regress attempts to implement the other behaviors faithfully.
|
||||
|
||||
### Character sets
|
||||
|
||||
tl;dr use UTF-8 (or ASCII) input and the 'u' flag, unless you are implementing a JavaScript engine and care about strict conformance.
|
||||
|
||||
To support JavaScript pre-Unicode semantics, regress supports multiple input forms on the `Regex` object. These are:
|
||||
|
||||
- **UTF-8**. The default (unsuffixed) form. Input is `&str`. This always decodes whole characters from the input string.
|
||||
- **ASCII**. Use the `*_ascii` family of functions on `Regex` if you know your input is ASCII. Input is still `&str`.
|
||||
- **UTF-16**. Use the `*_utf16` family of functions. Input is `&[u16]`. Characters are always decoded as UTF-16.
|
||||
- **UCS-2**. OG JavaScript. Use `*_ucs2` functions. Input is `&[u16]`. Surrogate pairs are split freely. Only use if you want to implement strict JS semantics.
|
||||
|
||||
Both the UTF-16 and UCS-2 forms require the Rust feature 'utf16' to be enabled. It is off by default.
|
||||
|
||||
### Fun Tools
|
||||
|
||||
The `regress-tool` binary can be used for some fun.
|
||||
|
||||
You can see how things get compiled with the `dump-phases` cli flag:
|
||||
|
||||
> cargo run 'x{3,4}' 'i' --dump-phases
|
||||
|
||||
You can run a little benchmark too, for example:
|
||||
|
||||
> cargo run --release -- 'abcd' --flags 'i' --bench ~/3200.txt
|
||||
|
||||
## OxideJS fork note
|
||||
|
||||
This directory is a vendored copy of upstream `regress 0.11.1` (source at
|
||||
`https://github.com/ridiculousfish/regress`, crate files from the crates.io
|
||||
registry, licenses preserved in `LICENSE-APACHE` / `LICENSE-MIT`), consumed via
|
||||
the workspace root `[patch.crates-io]` instead of the registry copy.
|
||||
|
||||
Single deviation from upstream: the Unicode script value table
|
||||
(`src/unicodetables.rs`) gains the `Unknown` (`Zzzz`) value — script and
|
||||
script-extensions interval sets covering the unassigned code points — plus its
|
||||
name/alias arm in the script-value parser. Upstream 0.11.1 lacks the value,
|
||||
which rejects spec-legal patterns such as `\p{Script=Unknown}`; upstream 0.12.0
|
||||
has it, but the 0.11.1 → 0.12.0 delta changes broader parser/matcher behavior
|
||||
this project does not want to absorb. All other files are unmodified upstream
|
||||
0.11.1.
|
||||
Vendored
+986
@@ -0,0 +1,986 @@
|
||||
use crate::classicalbacktrack;
|
||||
use crate::emit;
|
||||
use crate::exec;
|
||||
use crate::indexing;
|
||||
use crate::insn::CompiledRegex;
|
||||
use crate::optimizer;
|
||||
use crate::parse;
|
||||
use crate::types::MAX_CAPTURE_GROUPS;
|
||||
use std::iter::FusedIterator;
|
||||
|
||||
#[cfg(feature = "utf16")]
|
||||
use crate::{
|
||||
classicalbacktrack::MatchAttempter,
|
||||
indexing::{InputIndexer, Ucs2Input, Utf16Input},
|
||||
};
|
||||
|
||||
#[cfg(feature = "backend-pikevm")]
|
||||
use crate::pikevm;
|
||||
use crate::util::to_char_sat;
|
||||
|
||||
use core::{fmt, str::FromStr};
|
||||
#[cfg(feature = "std")]
|
||||
#[cfg(not(feature = "std"))]
|
||||
use {
|
||||
alloc::{string::String, vec::Vec},
|
||||
hashbrown::{HashMap, hash_map::Iter},
|
||||
};
|
||||
|
||||
pub use parse::Error;
|
||||
|
||||
/// Flags used to control regex parsing.
|
||||
/// The default flags are case-sensitive, not-multiline, and optimizing.
|
||||
#[derive(Debug, Copy, Clone, Default)]
|
||||
pub struct Flags {
|
||||
/// If set, make the regex case-insensitive.
|
||||
/// Equivalent to the 'i' flag in JavaScript.
|
||||
pub icase: bool,
|
||||
|
||||
/// If set, ^ and $ match at line separators, not just the input boundaries.
|
||||
/// Equivalent to the 'm' flag in JavaScript.
|
||||
pub multiline: bool,
|
||||
|
||||
/// If set, . matches at line separators as well as any other character.
|
||||
/// Equivalent to the 's' flag in JavaScript.
|
||||
pub dot_all: bool,
|
||||
|
||||
/// If set, disable regex IR passes.
|
||||
pub no_opt: bool,
|
||||
|
||||
/// If set, the regex is interpreted as a Unicode regex.
|
||||
/// Equivalent to the 'u' flag in JavaScript.
|
||||
pub unicode: bool,
|
||||
|
||||
/// If set, the regex is interpreted as a UnicodeSets regex.
|
||||
/// Equivalent to the 'v' flag in JavaScript.
|
||||
pub unicode_sets: bool,
|
||||
}
|
||||
|
||||
impl Flags {
|
||||
/// Construct a Flags from a Unicode codepoints iterator, using JavaScript field names.
|
||||
/// 'i' means to ignore case, 'm' means multiline, 'u' means unicode.
|
||||
/// Note the 'g' flag implies a stateful regex and is not supported.
|
||||
/// Other flags are not implemented and are ignored.
|
||||
#[inline]
|
||||
pub fn new<T: Iterator<Item = u32>>(chars: T) -> Self {
|
||||
let mut result = Self::default();
|
||||
for c in chars {
|
||||
match to_char_sat(c) {
|
||||
'm' => {
|
||||
result.multiline = true;
|
||||
}
|
||||
'i' => {
|
||||
result.icase = true;
|
||||
}
|
||||
's' => {
|
||||
result.dot_all = true;
|
||||
}
|
||||
'u' => {
|
||||
result.unicode = true;
|
||||
}
|
||||
'v' => {
|
||||
result.unicode_sets = true;
|
||||
}
|
||||
_ => {
|
||||
// Silently skip unsupported flags.
|
||||
}
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for Flags {
|
||||
/// Construct a Flags from a string, using JavaScript field names.
|
||||
///
|
||||
/// See also: [`Flags::new`].
|
||||
#[inline]
|
||||
fn from(s: &str) -> Self {
|
||||
Self::new(s.chars().map(u32::from))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Flags {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
if self.multiline {
|
||||
f.write_str("m")?;
|
||||
}
|
||||
if self.icase {
|
||||
f.write_str("i")?;
|
||||
}
|
||||
if self.dot_all {
|
||||
f.write_str("s")?;
|
||||
}
|
||||
if self.unicode {
|
||||
f.write_str("u")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Range is used to express the extent of a match, as indexes into the input
|
||||
/// string.
|
||||
pub type Range = core::ops::Range<usize>;
|
||||
|
||||
/// An iterator type which yields `Match`es found in a string.
|
||||
pub type Matches<'r, 't> = exec::Matches<backends::DefaultExecutor<'r, 't>>;
|
||||
|
||||
/// An iterator type which yields `Match`es found in a string, supporting ASCII
|
||||
/// only.
|
||||
pub type AsciiMatches<'r, 't> = exec::Matches<backends::DefaultAsciiExecutor<'r, 't>>;
|
||||
|
||||
/// A Match represents a portion of a string which was found to match a Regex.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Match {
|
||||
/// The total range of the match. Note this may be empty, if the regex
|
||||
/// matched an empty string.
|
||||
pub range: Range,
|
||||
|
||||
/// The list of captures. This has length equal to the number of capturing
|
||||
/// groups in the regex. For each capture, if the value is None, that group
|
||||
/// did not match (for example, it was in a not-taken branch of an
|
||||
/// alternation). If the value is Some, the group did match with the
|
||||
/// enclosed range.
|
||||
pub captures: Vec<Option<Range>>,
|
||||
|
||||
// A list of capture group names. This is either:
|
||||
// - Empty, if there were no named capture groups.
|
||||
// - A list of names with length `captures.len()`, corresponding to the
|
||||
// capture group names in order. Groups without names have an empty string.
|
||||
pub(crate) group_names: Box<[Box<str>]>,
|
||||
}
|
||||
|
||||
impl Match {
|
||||
/// Access a group by index, using the convention of Python's group()
|
||||
/// function. Index 0 is the total match, index 1 is the first capture
|
||||
/// group.
|
||||
#[inline]
|
||||
pub fn group(&self, idx: usize) -> Option<Range> {
|
||||
if idx == 0 {
|
||||
Some(self.range.clone())
|
||||
} else if idx <= self.captures.len() {
|
||||
self.captures[idx - 1].clone()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Access a named group by name.
|
||||
#[inline]
|
||||
pub fn named_group(&self, name: &str) -> Option<Range> {
|
||||
// Empty strings are used as sentinels to indicate unnamed group.
|
||||
if name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let pos = self.group_names.iter().position(|s| s.as_ref() == name)?;
|
||||
self.captures[pos].clone()
|
||||
}
|
||||
|
||||
/// Return an iterator over the named groups of a Match.
|
||||
#[inline]
|
||||
pub fn named_groups(&self) -> NamedGroups<'_> {
|
||||
NamedGroups::new(self)
|
||||
}
|
||||
|
||||
/// Returns the range over the starting and ending byte offsets of the match in the haystack.
|
||||
///
|
||||
/// This is a convenience function to work around
|
||||
/// the fact that Range does not support Copy.
|
||||
#[inline]
|
||||
pub fn range(&self) -> Range {
|
||||
self.range.clone()
|
||||
}
|
||||
|
||||
/// Returns the starting byte offset of the match in the haystack.
|
||||
#[inline]
|
||||
pub fn start(&self) -> usize {
|
||||
self.range.start
|
||||
}
|
||||
|
||||
/// Returns the ending byte offset of the match in the haystack.
|
||||
#[inline]
|
||||
pub fn end(&self) -> usize {
|
||||
self.range.end
|
||||
}
|
||||
|
||||
/// Returns the matched text as a string slice.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```rust
|
||||
/// use regress::Regex;
|
||||
///
|
||||
/// let re = Regex::new(r"\d+").unwrap();
|
||||
/// let text = "Price: $123";
|
||||
/// let m = re.find(text).unwrap();
|
||||
/// assert_eq!(m.as_str(text), "123");
|
||||
/// ```
|
||||
#[inline]
|
||||
pub fn as_str<'t>(&self, text: &'t str) -> &'t str {
|
||||
&text[self.range()]
|
||||
}
|
||||
|
||||
/// Return an iterator over a Match. The first returned value is the total
|
||||
/// match, and subsequent values represent the capture groups.
|
||||
#[inline]
|
||||
pub fn groups(&self) -> Groups<'_> {
|
||||
Groups::new(self)
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator over the capture groups of a [`Match`]
|
||||
///
|
||||
/// This struct is created by the [`groups`] method on [`Match`].
|
||||
///
|
||||
/// [`Match`]: ../struct.Match.html
|
||||
/// [`groups`]: ../struct.Match.html#method.groups
|
||||
#[derive(Clone)]
|
||||
pub struct Groups<'m> {
|
||||
mat: &'m Match,
|
||||
|
||||
// The next group index to return, where 0 references the total match.
|
||||
next_group_idx: usize,
|
||||
|
||||
// The maximum group index to return, with a +1 for the implicit total match.
|
||||
// For example, in a regex with 1 capture group, this will be 2.
|
||||
max: usize,
|
||||
}
|
||||
|
||||
impl<'m> Groups<'m> {
|
||||
#[inline]
|
||||
fn new(mat: &'m Match) -> Self {
|
||||
Self {
|
||||
mat,
|
||||
next_group_idx: 0,
|
||||
max: mat.captures.len() + 1, // +1 for the total match
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for Groups<'_> {
|
||||
type Item = Option<Range>;
|
||||
|
||||
#[inline]
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let i = self.next_group_idx;
|
||||
if i < self.max {
|
||||
self.next_group_idx += 1;
|
||||
Some(self.mat.group(i))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let size = self.max.saturating_sub(self.next_group_idx);
|
||||
(size, Some(size))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'m> ExactSizeIterator for Groups<'m> {}
|
||||
impl<'m> FusedIterator for Groups<'m> {}
|
||||
|
||||
/// An iterator over the named capture groups of a [`Match`]
|
||||
///
|
||||
/// This struct is created by the [`named_groups`] method on [`Match`].
|
||||
///
|
||||
/// [`Match`]: ../struct.Match.html
|
||||
/// [`named_groups`]: ../struct.Match.html#method.named_groups
|
||||
#[derive(Clone)]
|
||||
pub struct NamedGroups<'m> {
|
||||
mat: &'m Match,
|
||||
|
||||
// The next group name index to return.
|
||||
// Note unlike `Groups` this does NOT include the implicit total match.
|
||||
// That is, group 0 is the first capture group, NOT the total match.
|
||||
next_group_idx: usize,
|
||||
}
|
||||
|
||||
impl<'m> NamedGroups<'m> {
|
||||
#[inline]
|
||||
fn new(mat: &'m Match) -> Self {
|
||||
Self {
|
||||
mat,
|
||||
next_group_idx: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'m> Iterator for NamedGroups<'m> {
|
||||
type Item = (&'m str, Option<Range>);
|
||||
|
||||
#[inline]
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
// Increment next_group_idx until we find a non-empty name that we haven't seen yet.
|
||||
debug_assert!(self.next_group_idx <= self.mat.group_names.len());
|
||||
let end = self.mat.group_names.len();
|
||||
|
||||
loop {
|
||||
let mut idx = self.next_group_idx;
|
||||
// Skip empty names
|
||||
while idx < end && self.mat.group_names[idx].is_empty() {
|
||||
idx += 1;
|
||||
}
|
||||
if idx == end {
|
||||
return None;
|
||||
}
|
||||
|
||||
let name = self.mat.group_names[idx].as_ref();
|
||||
|
||||
// Check if we've already returned this name (by looking backwards)
|
||||
let already_seen = self.mat.group_names[..idx]
|
||||
.iter()
|
||||
.any(|n| n.as_ref() == name);
|
||||
|
||||
if already_seen {
|
||||
// Skip this duplicate and continue to next
|
||||
self.next_group_idx = idx + 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
// This is the first occurrence of this name. Find the best range value.
|
||||
// Prefer a Some value over None when there are duplicate names.
|
||||
let mut best_range = self.mat.captures[idx].clone();
|
||||
for check_idx in (idx + 1)..end {
|
||||
if self.mat.group_names[check_idx].as_ref() == name {
|
||||
// Found a duplicate name. Prefer a Some value over None.
|
||||
if best_range.is_none() && self.mat.captures[check_idx].is_some() {
|
||||
best_range = self.mat.captures[check_idx].clone();
|
||||
break; // Stop once we find a Some value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.next_group_idx = idx + 1;
|
||||
return Some((name, best_range));
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let size = self.mat.group_names[self.next_group_idx..]
|
||||
.iter()
|
||||
.filter(|s| !s.is_empty())
|
||||
.count();
|
||||
|
||||
(size, Some(size))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'m> ExactSizeIterator for NamedGroups<'m> {}
|
||||
impl<'m> FusedIterator for NamedGroups<'m> {}
|
||||
|
||||
/// A Regex is the compiled version of a pattern.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Regex {
|
||||
cr: CompiledRegex,
|
||||
}
|
||||
|
||||
impl From<CompiledRegex> for Regex {
|
||||
fn from(cr: CompiledRegex) -> Self {
|
||||
Self { cr }
|
||||
}
|
||||
}
|
||||
|
||||
impl Regex {
|
||||
/// Construct a regex by parsing `pattern` using the default flags.
|
||||
/// An Error may be returned if the syntax is invalid.
|
||||
/// Note that this is rather expensive; prefer to cache a Regex which is
|
||||
/// intended to be used more than once.
|
||||
#[inline]
|
||||
pub fn new(pattern: &str) -> Result<Regex, Error> {
|
||||
Self::with_flags(pattern, Flags::default())
|
||||
}
|
||||
|
||||
/// Construct a regex by parsing `pattern` with `flags`.
|
||||
/// An Error may be returned if the syntax is invalid.
|
||||
//
|
||||
/// Note it is preferable to cache a Regex which is intended to be used more
|
||||
/// than once, as the parse may be expensive. For example:
|
||||
#[inline]
|
||||
pub fn with_flags<F>(pattern: &str, flags: F) -> Result<Regex, Error>
|
||||
where
|
||||
F: Into<Flags>,
|
||||
{
|
||||
Self::from_unicode(pattern.chars().map(u32::from), flags)
|
||||
}
|
||||
|
||||
/// Construct a regex by parsing `pattern` with `flags`, where
|
||||
/// `pattern` is an iterator of `u32` Unicode codepoints.
|
||||
/// An Error may be returned if the syntax is invalid.
|
||||
/// This allows parsing regular expressions from exotic strings in
|
||||
/// other encodings, such as UTF-16 or UTF-32.
|
||||
pub fn from_unicode<I, F>(pattern: I, flags: F) -> Result<Regex, Error>
|
||||
where
|
||||
I: Iterator<Item = u32> + Clone,
|
||||
F: Into<Flags>,
|
||||
{
|
||||
let flags = flags.into();
|
||||
let mut ire = parse::try_parse(pattern, flags)?;
|
||||
if !flags.no_opt {
|
||||
optimizer::optimize(&mut ire);
|
||||
}
|
||||
let cr = emit::emit(&ire);
|
||||
Ok(Regex { cr })
|
||||
}
|
||||
|
||||
/// Searches `text` to find the first match.
|
||||
#[inline]
|
||||
pub fn find(&self, text: &str) -> Option<Match> {
|
||||
self.find_iter(text).next()
|
||||
}
|
||||
|
||||
/// Searches `text`, returning an iterator over non-overlapping matches.
|
||||
/// Note that the resulting Iterator borrows both the regex `'r` and the
|
||||
/// input string as `'t`.
|
||||
#[inline]
|
||||
pub fn find_iter<'r, 't>(&'r self, text: &'t str) -> Matches<'r, 't> {
|
||||
self.find_from(text, 0)
|
||||
}
|
||||
|
||||
/// Returns an iterator for matches found in 'text' starting at byte index
|
||||
/// `start`. Note this may be different from passing a sliced `text` in
|
||||
/// the case of lookbehind assertions.
|
||||
/// Example:
|
||||
///
|
||||
/// ```rust
|
||||
/// use regress::Regex;
|
||||
/// let text = "xyxy";
|
||||
/// let re = Regex::new(r"(?<=x)y").unwrap();
|
||||
/// let t1 = re.find(&text[1..]).unwrap().range();
|
||||
/// assert!(t1 == (2..3));
|
||||
/// let t2 = re.find_from(text, 1).next().unwrap().range();
|
||||
/// assert!(t2 == (1..2));
|
||||
/// ```
|
||||
#[inline]
|
||||
pub fn find_from<'r, 't>(&'r self, text: &'t str, start: usize) -> Matches<'r, 't> {
|
||||
backends::find(self, text, start)
|
||||
}
|
||||
|
||||
/// Searches `text` to find the first match.
|
||||
/// The input text is expected to be ascii-only: only ASCII case-folding is
|
||||
/// supported.
|
||||
#[inline]
|
||||
pub fn find_ascii(&self, text: &str) -> Option<Match> {
|
||||
self.find_iter_ascii(text).next()
|
||||
}
|
||||
|
||||
/// Searches `text`, returning an iterator over non-overlapping matches.
|
||||
/// The input text is expected to be ascii-only: only ASCII case-folding is
|
||||
/// supported.
|
||||
#[inline]
|
||||
pub fn find_iter_ascii<'r, 't>(&'r self, text: &'t str) -> AsciiMatches<'r, 't> {
|
||||
self.find_from_ascii(text, 0)
|
||||
}
|
||||
|
||||
/// Returns an iterator for matches found in 'text' starting at byte index
|
||||
/// `start`.
|
||||
#[inline]
|
||||
pub fn find_from_ascii<'r, 't>(&'r self, text: &'t str, start: usize) -> AsciiMatches<'r, 't> {
|
||||
backends::find(self, text, start)
|
||||
}
|
||||
|
||||
/// Returns an iterator for matches found in 'text' starting at index `start`.
|
||||
#[cfg(feature = "utf16")]
|
||||
pub fn find_from_utf16<'r, 't>(
|
||||
&'r self,
|
||||
text: &'t [u16],
|
||||
start: usize,
|
||||
) -> exec::Matches<super::classicalbacktrack::BacktrackExecutor<'r, indexing::Utf16Input<'t>>>
|
||||
{
|
||||
let input = Utf16Input::new(text, self.cr.flags.unicode);
|
||||
exec::Matches::new(
|
||||
super::classicalbacktrack::BacktrackExecutor::new(
|
||||
input,
|
||||
MatchAttempter::new(&self.cr, input.left_end()),
|
||||
),
|
||||
start,
|
||||
)
|
||||
}
|
||||
|
||||
/// Returns an iterator for matches found in 'text' starting at index `start`.
|
||||
#[cfg(feature = "utf16")]
|
||||
pub fn find_from_ucs2<'r, 't>(
|
||||
&'r self,
|
||||
text: &'t [u16],
|
||||
start: usize,
|
||||
) -> exec::Matches<super::classicalbacktrack::BacktrackExecutor<'r, indexing::Ucs2Input<'t>>>
|
||||
{
|
||||
let input = Ucs2Input::new(text, self.cr.flags.unicode);
|
||||
exec::Matches::new(
|
||||
super::classicalbacktrack::BacktrackExecutor::new(
|
||||
input,
|
||||
MatchAttempter::new(&self.cr, input.left_end()),
|
||||
),
|
||||
start,
|
||||
)
|
||||
}
|
||||
|
||||
/// Replaces the first match of the regex in `text` with the replacement string.
|
||||
///
|
||||
/// The replacement string may contain capture group references in the form `$1`, `$2`, etc.,
|
||||
/// where `$1` refers to the first capture group, `$2` to the second, and so on.
|
||||
/// `$0` refers to the entire match. Use `$$` to insert a literal `$`.
|
||||
///
|
||||
/// If no match is found, the original text is returned unchanged.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```rust
|
||||
/// use regress::Regex;
|
||||
///
|
||||
/// let re = Regex::new(r"(\w+)\s+(\w+)").unwrap();
|
||||
/// let result = re.replace("hello world", "$2 $1");
|
||||
/// assert_eq!(result, "world hello");
|
||||
///
|
||||
/// let re = Regex::new(r"(\d{4})-(\d{2})-(\d{2})").unwrap();
|
||||
/// let result = re.replace("2023-12-25", "$2/$3/$1");
|
||||
/// assert_eq!(result, "12/25/2023");
|
||||
/// ```
|
||||
pub fn replace(&self, text: &str, replacement: &str) -> String {
|
||||
match self.find(text) {
|
||||
Some(m) => {
|
||||
let mut result = String::with_capacity(text.len());
|
||||
result.push_str(&text[..m.start()]);
|
||||
self.expand_replacement(&m, text, replacement, &mut result);
|
||||
result.push_str(&text[m.end()..]);
|
||||
result
|
||||
}
|
||||
None => text.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces all matches of the regex in `text` with the replacement string.
|
||||
///
|
||||
/// The replacement string may contain capture group references in the form `$1`, `$2`, etc.,
|
||||
/// where `$1` refers to the first capture group, `$2` to the second, and so on.
|
||||
/// `$0` refers to the entire match. Use `$$` to insert a literal `$`.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```rust
|
||||
/// use regress::Regex;
|
||||
///
|
||||
/// let re = Regex::new(r"(\w+)\s+(\w+)").unwrap();
|
||||
/// let result = re.replace_all("hello world foo bar", "$2-$1");
|
||||
/// assert_eq!(result, "world-hello bar-foo");
|
||||
///
|
||||
/// let re = Regex::new(r"\b(\w)(\w+)").unwrap();
|
||||
/// let result = re.replace_all("hello world", "$1.$2");
|
||||
/// assert_eq!(result, "h.ello w.orld");
|
||||
/// ```
|
||||
pub fn replace_all(&self, text: &str, replacement: &str) -> String {
|
||||
let mut result = String::with_capacity(text.len());
|
||||
let mut last_end = 0;
|
||||
|
||||
for m in self.find_iter(text) {
|
||||
result.push_str(&text[last_end..m.start()]);
|
||||
self.expand_replacement(&m, text, replacement, &mut result);
|
||||
last_end = m.end();
|
||||
}
|
||||
|
||||
result.push_str(&text[last_end..]);
|
||||
result
|
||||
}
|
||||
|
||||
/// Replaces the first match of the regex in `text` using a closure.
|
||||
///
|
||||
/// The closure receives a `&Match` and should return the replacement string.
|
||||
/// This is useful for dynamic replacements that depend on the match details.
|
||||
///
|
||||
/// If no match is found, the original text is returned unchanged.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```rust
|
||||
/// use regress::Regex;
|
||||
///
|
||||
/// let re = Regex::new(r"\d+").unwrap();
|
||||
/// let text = "Price: $123";
|
||||
/// let result = re.replace_with(text, |m| {
|
||||
/// let num: i32 = m.as_str(text).parse().unwrap();
|
||||
/// format!("{}", num * 2)
|
||||
/// });
|
||||
/// assert_eq!(result, "Price: $246");
|
||||
/// ```
|
||||
pub fn replace_with<F>(&self, text: &str, replacement: F) -> String
|
||||
where
|
||||
F: FnOnce(&Match) -> String,
|
||||
{
|
||||
match self.find(text) {
|
||||
Some(m) => {
|
||||
let mut result = String::with_capacity(text.len());
|
||||
result.push_str(&text[..m.start()]);
|
||||
result.push_str(&replacement(&m));
|
||||
result.push_str(&text[m.end()..]);
|
||||
result
|
||||
}
|
||||
None => text.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces all matches of the regex in `text` using a closure.
|
||||
///
|
||||
/// The closure receives a `&Match` and should return the replacement string.
|
||||
/// This is useful for dynamic replacements that depend on the match details.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```rust
|
||||
/// use regress::Regex;
|
||||
///
|
||||
/// let re = Regex::new(r"\d+").unwrap();
|
||||
/// let text = "Items: 5, 10, 15";
|
||||
/// let result = re.replace_all_with(text, |m| {
|
||||
/// let num: i32 = m.as_str(text).parse().unwrap();
|
||||
/// format!("[{}]", num * 10)
|
||||
/// });
|
||||
/// assert_eq!(result, "Items: [50], [100], [150]");
|
||||
/// ```
|
||||
pub fn replace_all_with<F>(&self, text: &str, replacement: F) -> String
|
||||
where
|
||||
F: Fn(&Match) -> String,
|
||||
{
|
||||
let mut result = String::with_capacity(text.len());
|
||||
let mut last_end = 0;
|
||||
|
||||
for m in self.find_iter(text) {
|
||||
result.push_str(&text[last_end..m.start()]);
|
||||
result.push_str(&replacement(&m));
|
||||
last_end = m.end();
|
||||
}
|
||||
|
||||
result.push_str(&text[last_end..]);
|
||||
result
|
||||
}
|
||||
|
||||
/// Helper method to expand replacement strings with capture group substitutions.
|
||||
fn expand_replacement(&self, m: &Match, text: &str, replacement: &str, output: &mut String) {
|
||||
let mut chars = replacement.chars().peekable();
|
||||
|
||||
while let Some(ch) = chars.next() {
|
||||
if ch == '$' {
|
||||
match chars.peek() {
|
||||
Some('$') => {
|
||||
// $$ -> literal $
|
||||
chars.next();
|
||||
output.push('$');
|
||||
}
|
||||
Some(&digit) if digit.is_ascii_digit() => {
|
||||
// Parse the group number
|
||||
let mut group_num = 0;
|
||||
while let Some(&digit) = chars.peek() {
|
||||
if digit.is_ascii_digit() {
|
||||
chars.next();
|
||||
group_num = group_num * 10 + (digit as u32 - '0' as u32) as usize;
|
||||
// Limit to reasonable group numbers to avoid overflow
|
||||
if group_num > MAX_CAPTURE_GROUPS {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Get the matched text for this group
|
||||
if let Some(range) = m.group(group_num) {
|
||||
output.push_str(&text[range]);
|
||||
}
|
||||
// If group doesn't exist or didn't match, add nothing
|
||||
}
|
||||
Some('{') => {
|
||||
// Handle ${name} syntax for named groups
|
||||
chars.next(); // consume '{'
|
||||
let mut name = String::new();
|
||||
let mut found_closing_brace = false;
|
||||
|
||||
for ch in chars.by_ref() {
|
||||
if ch == '}' {
|
||||
found_closing_brace = true;
|
||||
break;
|
||||
}
|
||||
name.push(ch);
|
||||
}
|
||||
|
||||
if found_closing_brace {
|
||||
if let Some(range) = m.named_group(&name) {
|
||||
output.push_str(&text[range]);
|
||||
}
|
||||
} else {
|
||||
// Malformed ${...}, treat as literal
|
||||
output.push_str("${");
|
||||
output.push_str(&name);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Just a $ at end or followed by non-digit
|
||||
output.push('$');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
output.push(ch);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for Regex {
|
||||
type Err = Error;
|
||||
|
||||
/// Attempts to parse a string into a regular expression
|
||||
#[inline]
|
||||
fn from_str(s: &str) -> Result<Self, Error> {
|
||||
Self::new(s)
|
||||
}
|
||||
}
|
||||
|
||||
// Pattern trait implementation for str::find, str::contains, etc.
|
||||
#[cfg(feature = "pattern")]
|
||||
mod pattern_impl {
|
||||
use super::*;
|
||||
use core::str::pattern::{Pattern, ReverseSearcher, SearchStep, Searcher};
|
||||
|
||||
/// A searcher for a regex pattern.
|
||||
pub struct RegexSearcher<'r, 't> {
|
||||
haystack: &'t str,
|
||||
regex: &'r Regex,
|
||||
current_pos: usize,
|
||||
done: bool,
|
||||
// For reverse searching
|
||||
reverse_pos: usize,
|
||||
reverse_done: bool,
|
||||
}
|
||||
|
||||
impl<'r, 't> RegexSearcher<'r, 't> {
|
||||
fn new(regex: &'r Regex, haystack: &'t str) -> Self {
|
||||
Self {
|
||||
haystack,
|
||||
regex,
|
||||
current_pos: 0,
|
||||
done: false,
|
||||
reverse_pos: haystack.len(),
|
||||
reverse_done: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn find_last_match_before(&self, pos: usize) -> Option<super::Match> {
|
||||
// Find all matches up to the given position and return the last one
|
||||
let mut last_match = None;
|
||||
for m in self.regex.find_from(self.haystack, 0) {
|
||||
if m.end() <= pos {
|
||||
last_match = Some(m);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
last_match
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl<'r, 't> Searcher<'t> for RegexSearcher<'r, 't> {
|
||||
fn haystack(&self) -> &'t str {
|
||||
self.haystack
|
||||
}
|
||||
|
||||
fn next(&mut self) -> SearchStep {
|
||||
if self.done {
|
||||
return SearchStep::Done;
|
||||
}
|
||||
|
||||
// Try to find the next match starting from current position
|
||||
if let Some(m) = self.regex.find_from(self.haystack, self.current_pos).next() {
|
||||
let match_start = m.start();
|
||||
let match_end = m.end();
|
||||
|
||||
// Handle any gap between current position and match start
|
||||
if self.current_pos < match_start {
|
||||
let reject_end = match_start;
|
||||
let reject_start = self.current_pos;
|
||||
self.current_pos = match_start;
|
||||
return SearchStep::Reject(reject_start, reject_end);
|
||||
}
|
||||
|
||||
// Return the match
|
||||
self.current_pos = match_end;
|
||||
|
||||
// Handle zero-width matches to avoid infinite loops
|
||||
if match_start == match_end {
|
||||
// For zero-width matches, we need to advance at least one byte
|
||||
// to avoid infinite loops
|
||||
if match_end < self.haystack.len() {
|
||||
// Find the next character boundary
|
||||
let mut next_pos = match_end + 1;
|
||||
while next_pos < self.haystack.len()
|
||||
&& !self.haystack.is_char_boundary(next_pos)
|
||||
{
|
||||
next_pos += 1;
|
||||
}
|
||||
self.current_pos = next_pos;
|
||||
} else {
|
||||
// We're at the end of the string
|
||||
self.done = true;
|
||||
}
|
||||
}
|
||||
|
||||
SearchStep::Match(match_start, match_end)
|
||||
} else {
|
||||
// No more matches, reject remaining text if any
|
||||
if self.current_pos < self.haystack.len() {
|
||||
let reject_start = self.current_pos;
|
||||
let reject_end = self.haystack.len();
|
||||
self.current_pos = self.haystack.len();
|
||||
self.done = true;
|
||||
SearchStep::Reject(reject_start, reject_end)
|
||||
} else {
|
||||
self.done = true;
|
||||
SearchStep::Done
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl<'r, 't> ReverseSearcher<'t> for RegexSearcher<'r, 't> {
|
||||
fn next_back(&mut self) -> SearchStep {
|
||||
if self.reverse_done {
|
||||
return SearchStep::Done;
|
||||
}
|
||||
|
||||
// Try to find the last match before current reverse position
|
||||
if let Some(m) = self.find_last_match_before(self.reverse_pos) {
|
||||
let match_start = m.start();
|
||||
let match_end = m.end();
|
||||
|
||||
// Handle any gap between match end and current reverse position
|
||||
if match_end < self.reverse_pos {
|
||||
let reject_start = match_end;
|
||||
let reject_end = self.reverse_pos;
|
||||
self.reverse_pos = match_end;
|
||||
return SearchStep::Reject(reject_start, reject_end);
|
||||
}
|
||||
|
||||
// Return the match
|
||||
self.reverse_pos = match_start;
|
||||
|
||||
// Handle zero-width matches
|
||||
if match_start == match_end {
|
||||
// For zero-width matches, move back by one character
|
||||
if match_start > 0 {
|
||||
let mut prev_pos = match_start - 1;
|
||||
while prev_pos > 0 && !self.haystack.is_char_boundary(prev_pos) {
|
||||
prev_pos -= 1;
|
||||
}
|
||||
self.reverse_pos = prev_pos;
|
||||
} else {
|
||||
// We're at the beginning of the string
|
||||
self.reverse_done = true;
|
||||
}
|
||||
}
|
||||
|
||||
SearchStep::Match(match_start, match_end)
|
||||
} else {
|
||||
// No more matches, reject remaining text if any
|
||||
if self.reverse_pos > 0 {
|
||||
let reject_start = 0;
|
||||
let reject_end = self.reverse_pos;
|
||||
self.reverse_pos = 0;
|
||||
self.reverse_done = true;
|
||||
SearchStep::Reject(reject_start, reject_end)
|
||||
} else {
|
||||
self.reverse_done = true;
|
||||
SearchStep::Done
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'r> Pattern for &'r Regex {
|
||||
type Searcher<'a> = RegexSearcher<'r, 'a>;
|
||||
|
||||
fn into_searcher(self, haystack: &str) -> Self::Searcher<'_> {
|
||||
RegexSearcher::new(self, haystack)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "pattern")]
|
||||
pub use pattern_impl::*;
|
||||
|
||||
// Support for using regress with different regex backends.
|
||||
// Currently there is only the classical backtracking, and PikeVM.
|
||||
#[doc(hidden)]
|
||||
pub mod backends {
|
||||
use super::Regex;
|
||||
use super::exec;
|
||||
use super::indexing;
|
||||
pub use crate::emit::emit;
|
||||
pub use crate::optimizer::optimize;
|
||||
pub use crate::parse::try_parse;
|
||||
|
||||
/// An Executor using the classical backtracking algorithm.
|
||||
pub type BacktrackExecutor<'r, 't> =
|
||||
super::classicalbacktrack::BacktrackExecutor<'r, indexing::Utf8Input<'t>>;
|
||||
|
||||
/// A Executor using the PikeVM executor.
|
||||
#[cfg(feature = "backend-pikevm")]
|
||||
pub type PikeVMExecutor<'r, 't> = super::pikevm::PikeVMExecutor<'r, indexing::Utf8Input<'t>>;
|
||||
|
||||
/// An alias type to the default Executor.
|
||||
pub type DefaultExecutor<'r, 't> = BacktrackExecutor<'r, 't>;
|
||||
|
||||
/// An alias type to the default executor's ASCII form.
|
||||
pub type DefaultAsciiExecutor<'r, 't> =
|
||||
<DefaultExecutor<'r, 't> as exec::Executor<'r, 't>>::AsAscii;
|
||||
|
||||
/// Searches `text`, returning an iterator over non-overlapping matches.
|
||||
pub fn find<'r, 't, Executor: exec::Executor<'r, 't>>(
|
||||
re: &'r Regex,
|
||||
text: &'t str,
|
||||
start: usize,
|
||||
) -> exec::Matches<Executor> {
|
||||
exec::Matches::new(Executor::new(&re.cr, text), start)
|
||||
}
|
||||
|
||||
/// Searches `text`, returning an iterator over non-overlapping matches.
|
||||
/// This is a convenience method to avoid E0223.
|
||||
pub fn find_ascii<'r, 't, Executor: exec::Executor<'r, 't>>(
|
||||
re: &'r Regex,
|
||||
text: &'t str,
|
||||
start: usize,
|
||||
) -> exec::Matches<Executor::AsAscii> {
|
||||
find::<Executor::AsAscii>(re, text, start)
|
||||
}
|
||||
}
|
||||
|
||||
/// Escapes all special regex characters in a string to make it a literal match.
|
||||
///
|
||||
/// This function takes a string and returns a new string with all special
|
||||
/// regex characters escaped with backslashes, so the resulting string can be
|
||||
/// used as a literal pattern in a regular expression.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// use regress::escape;
|
||||
///
|
||||
/// let escaped = escape("Hello. How are you?");
|
||||
/// assert_eq!(escaped, "Hello\\. How are you\\?");
|
||||
///
|
||||
/// let escaped = escape("$100 + tax (15%)");
|
||||
/// assert_eq!(escaped, "\\$100 \\+ tax \\(15%\\)");
|
||||
/// ```
|
||||
pub fn escape(text: &str) -> String {
|
||||
let mut result = String::with_capacity(text.len());
|
||||
|
||||
for c in text.chars() {
|
||||
match c {
|
||||
// Characters that have special meaning in regex and need escaping
|
||||
'\\' | '^' | '$' | '.' | '|' | '?' | '*' | '+' | '(' | ')' | '[' | ']' | '{' | '}' => {
|
||||
result.push('\\');
|
||||
result.push(c);
|
||||
}
|
||||
// All other characters are literal
|
||||
_ => result.push(c),
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
+389
@@ -0,0 +1,389 @@
|
||||
use crate::insn::MAX_CHAR_SET_LENGTH;
|
||||
use core::fmt;
|
||||
extern crate memchr;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
|
||||
/// Facilities for searching bytes.
|
||||
pub trait ByteSearcher {
|
||||
/// Search for ourselves in a slice of bytes.
|
||||
/// The length of the slice is unspecified and may be 0.
|
||||
/// \return the next index of ourselves in the slice, or None.
|
||||
fn find_in(&self, rhs: &[u8]) -> Option<usize>;
|
||||
}
|
||||
|
||||
impl ByteSearcher for [u8; 1] {
|
||||
#[inline(always)]
|
||||
fn find_in(&self, rhs: &[u8]) -> Option<usize> {
|
||||
memchr::memchr(self[0], rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl ByteSearcher for [u8; 2] {
|
||||
#[inline(always)]
|
||||
fn find_in(&self, rhs: &[u8]) -> Option<usize> {
|
||||
memchr::memchr2(self[0], self[1], rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl ByteSearcher for [u8; 3] {
|
||||
#[inline(always)]
|
||||
fn find_in(&self, rhs: &[u8]) -> Option<usize> {
|
||||
memchr::memchr3(self[0], self[1], self[2], rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl ByteSearcher for memchr::memmem::Finder<'_> {
|
||||
fn find_in(&self, rhs: &[u8]) -> Option<usize> {
|
||||
self.find(rhs)
|
||||
}
|
||||
}
|
||||
|
||||
/// A ByteSet is any set of bytes.
|
||||
pub trait ByteSet {
|
||||
/// \return whether the ByteSet contains the byte.
|
||||
fn contains(&self, b: u8) -> bool;
|
||||
}
|
||||
|
||||
/// A ByteArraySet wraps a small array and uses linear equality.
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
#[repr(align(4))]
|
||||
pub struct ByteArraySet<ArraySet: SmallArraySet>(pub ArraySet);
|
||||
|
||||
/// Cover over contains() to avoid bumping into native contains call.
|
||||
impl<ArraySet: SmallArraySet> ByteArraySet<ArraySet> {
|
||||
#[inline(always)]
|
||||
pub fn contains(self, b: u8) -> bool {
|
||||
self.0.contains(b)
|
||||
}
|
||||
}
|
||||
|
||||
impl<ArraySet: SmallArraySet> ByteSearcher for ByteArraySet<ArraySet> {
|
||||
#[inline(always)]
|
||||
fn find_in(&self, rhs: &[u8]) -> Option<usize> {
|
||||
self.0.find_in(rhs)
|
||||
}
|
||||
}
|
||||
|
||||
/// A SmallArraySet is a set implemented as a small byte array.
|
||||
pub trait SmallArraySet: Copy {
|
||||
fn contains(self, b: u8) -> bool;
|
||||
|
||||
fn find_in(self, rhs: &[u8]) -> Option<usize>;
|
||||
}
|
||||
|
||||
// Beware: Rust is cranky about loop unrolling.
|
||||
// Do not try to be too clever here.
|
||||
impl SmallArraySet for [u8; 2] {
|
||||
#[inline(always)]
|
||||
fn contains(self, b: u8) -> bool {
|
||||
b == self[0] || b == self[1]
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn find_in(self, rhs: &[u8]) -> Option<usize> {
|
||||
memchr::memchr2(self[0], self[1], rhs)
|
||||
}
|
||||
}
|
||||
impl SmallArraySet for [u8; 3] {
|
||||
#[inline(always)]
|
||||
fn contains(self, b: u8) -> bool {
|
||||
b == self[0] || b == self[1] || b == self[2]
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn find_in(self, rhs: &[u8]) -> Option<usize> {
|
||||
memchr::memchr3(self[0], self[1], self[2], rhs)
|
||||
}
|
||||
}
|
||||
impl SmallArraySet for [u8; 4] {
|
||||
#[inline(always)]
|
||||
fn contains(self, b: u8) -> bool {
|
||||
b == self[0] || b == self[1] || b == self[2] || b == self[3]
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn find_in(self, rhs: &[u8]) -> Option<usize> {
|
||||
// TODO.
|
||||
for (idx, byte) in rhs.iter().enumerate() {
|
||||
if self.contains(*byte) {
|
||||
return Some(idx);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
// CharSet helper. Avoid branching in the loop to get good unrolling.
|
||||
#[allow(unused_parens)]
|
||||
#[inline(always)]
|
||||
pub fn charset_contains(set: &[u32; MAX_CHAR_SET_LENGTH], c: u32) -> bool {
|
||||
let mut result = false;
|
||||
for &v in set.iter() {
|
||||
result |= (v == c);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// A helper function for formatting bitmaps, using - ranges.
|
||||
fn format_bitmap<Func>(name: &str, f: &mut fmt::Formatter<'_>, contains: Func) -> fmt::Result
|
||||
where
|
||||
Func: Fn(u8) -> bool,
|
||||
{
|
||||
write!(f, "{}[", name)?;
|
||||
let mut idx = 0;
|
||||
let mut maybe_space = "";
|
||||
while idx <= 256 {
|
||||
// Compute the next value not contained.
|
||||
let mut end = idx;
|
||||
while end <= 256 && contains(end as u8) {
|
||||
end += 1;
|
||||
}
|
||||
match end - idx {
|
||||
0 => (),
|
||||
1 => write!(f, "{}{}", maybe_space, idx)?,
|
||||
_ => write!(f, "{}{}-{}", maybe_space, idx, end - 1)?,
|
||||
};
|
||||
if end > idx {
|
||||
maybe_space = " ";
|
||||
}
|
||||
idx = end + 1
|
||||
}
|
||||
write!(f, "]")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A bitmap covering ASCII characters.
|
||||
#[derive(Default, Copy, Clone)]
|
||||
#[repr(align(4))]
|
||||
pub struct AsciiBitmap(pub [u8; 16]);
|
||||
|
||||
impl AsciiBitmap {
|
||||
/// Set a byte val in this bitmap.
|
||||
#[inline(always)]
|
||||
pub fn set(&mut self, val: u8) {
|
||||
debug_assert!(val <= 127, "Value should be ASCII");
|
||||
self.0[(val >> 3) as usize] |= 1 << (val & 0x7);
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for AsciiBitmap {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
format_bitmap("AsciiBitmap", f, |v| self.contains(v))
|
||||
}
|
||||
}
|
||||
|
||||
impl ByteSet for AsciiBitmap {
|
||||
/// \return whether this bitmap contains a given value.
|
||||
/// The value does NOT have to be ASCII.
|
||||
#[inline(always)]
|
||||
fn contains(&self, val: u8) -> bool {
|
||||
// Delicate tricks to avoid branches.
|
||||
// In general we want to compute the byte via /8, and then mask into the
|
||||
// byte. But if the value is not ASCII then the byte could be too large.
|
||||
// So mask off the MSB so that the byte is always in range.
|
||||
let byte = (val & 0x7F) >> 3;
|
||||
let bit = val & 0x7;
|
||||
|
||||
// Now probe the bitmap. If our sign bit was set, we want the mask to be 0;
|
||||
// otherwise we want to set only the 'bit' offset.
|
||||
// Invert the sign bit and reuse it.
|
||||
let mask = ((val >> 7) ^ 1) << bit;
|
||||
|
||||
// Probe the bitmap. We expect the compiler to elide the bounds check.
|
||||
(self.0[byte as usize] & mask) != 0
|
||||
}
|
||||
}
|
||||
|
||||
/// A bitmap covering all bytes.
|
||||
#[derive(Default, Copy, Clone, PartialEq, Eq)]
|
||||
#[repr(align(4))]
|
||||
pub struct ByteBitmap([u16; 16]);
|
||||
|
||||
// TODO: the codegen here is pretty horrible; LLVM is emitting a sequence of
|
||||
// halfword instructions. Consider using a union?
|
||||
impl ByteBitmap {
|
||||
/// Construct from a sequence of bytes.
|
||||
pub fn new(bytes: &[u8]) -> ByteBitmap {
|
||||
let mut bb = ByteBitmap::default();
|
||||
for &b in bytes {
|
||||
bb.set(b)
|
||||
}
|
||||
bb
|
||||
}
|
||||
|
||||
/// \return whether this bitmap contains a given byte val.
|
||||
#[inline(always)]
|
||||
pub fn contains(&self, val: u8) -> bool {
|
||||
let byte = val >> 4;
|
||||
let bit = val & 0xF;
|
||||
(self.0[byte as usize] & (1 << bit)) != 0
|
||||
}
|
||||
|
||||
/// Set a bit in this bitmap.
|
||||
#[inline(always)]
|
||||
pub fn set(&mut self, val: u8) {
|
||||
let byte = val >> 4;
|
||||
let bit = val & 0xF;
|
||||
self.0[byte as usize] |= 1 << bit;
|
||||
}
|
||||
|
||||
/// Update ourselves from another bitmap, in place.
|
||||
pub fn bitor(&mut self, rhs: &ByteBitmap) {
|
||||
for idx in 0..self.0.len() {
|
||||
self.0[idx] |= rhs.0[idx];
|
||||
}
|
||||
}
|
||||
|
||||
/// Invert our bits, in place.
|
||||
pub fn bitnot(&mut self) -> &mut Self {
|
||||
for val in self.0.iter_mut() {
|
||||
*val = !*val;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Count number of set bits.
|
||||
pub fn count_bits(&self) -> u32 {
|
||||
self.0.iter().map(|v| v.count_ones()).sum()
|
||||
}
|
||||
|
||||
/// Return ourselves as an array of a fixed length.
|
||||
/// Panics if the array is not large enough.
|
||||
#[allow(clippy::wrong_self_convention)]
|
||||
#[inline(always)]
|
||||
pub fn as_array<const N: usize>(&self) -> [u8; N] {
|
||||
let mut array = [0u8; N];
|
||||
let mut idx = 0;
|
||||
for byte in 0..=255 {
|
||||
if self.contains(byte) {
|
||||
array[idx] = byte;
|
||||
idx += 1;
|
||||
}
|
||||
}
|
||||
array
|
||||
}
|
||||
|
||||
/// \return the index of the first byte in the slice that is present in this
|
||||
/// bitmap, using some unsafe tricks.
|
||||
#[inline(always)]
|
||||
fn unsafe_find_in_slice(&self, bytes: &[u8]) -> Option<usize> {
|
||||
type Chunk = u32;
|
||||
let bm = &self.0;
|
||||
|
||||
let mut offset = 0;
|
||||
let (prefix, body, suffix) = unsafe { bytes.align_to::<Chunk>() };
|
||||
for &byte in prefix.iter() {
|
||||
if self.contains(byte) {
|
||||
return Some(offset);
|
||||
}
|
||||
offset += 1;
|
||||
}
|
||||
|
||||
for &chunk in body {
|
||||
// Use LE. Here index 0 is the earliest address.
|
||||
let byte_idxs = ((chunk >> 4) & 0x0F0F0F0F).to_le_bytes();
|
||||
let bit_idxs = (chunk & 0x0F0F0F0F).to_le_bytes();
|
||||
if (bm[byte_idxs[0] as usize] & (1 << bit_idxs[0])) != 0 {
|
||||
return Some(offset);
|
||||
}
|
||||
if (bm[byte_idxs[1] as usize] & (1 << bit_idxs[1])) != 0 {
|
||||
return Some(offset + 1);
|
||||
}
|
||||
if (bm[byte_idxs[2] as usize] & (1 << bit_idxs[2])) != 0 {
|
||||
return Some(offset + 2);
|
||||
}
|
||||
if (bm[byte_idxs[3] as usize] & (1 << bit_idxs[3])) != 0 {
|
||||
return Some(offset + 3);
|
||||
}
|
||||
offset += 4;
|
||||
}
|
||||
|
||||
for &byte in suffix.iter() {
|
||||
if self.contains(byte) {
|
||||
return Some(offset);
|
||||
}
|
||||
offset += 1;
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl ByteSearcher for ByteBitmap {
|
||||
#[inline(always)]
|
||||
fn find_in(&self, bytes: &[u8]) -> Option<usize> {
|
||||
if cfg!(feature = "prohibit-unsafe") {
|
||||
for (idx, byte) in bytes.iter().enumerate() {
|
||||
if self.contains(*byte) {
|
||||
return Some(idx);
|
||||
}
|
||||
}
|
||||
None
|
||||
} else {
|
||||
self.unsafe_find_in_slice(bytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ByteBitmap {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
format_bitmap("ByteBitmap", f, |v| self.contains(v))
|
||||
}
|
||||
}
|
||||
|
||||
/// A trivial ByteSearcher corresponding to the empty string.
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct EmptyString {}
|
||||
|
||||
impl ByteSearcher for EmptyString {
|
||||
#[inline(always)]
|
||||
fn find_in(&self, _bytes: &[u8]) -> Option<usize> {
|
||||
Some(0)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_bitmap(bytes: &[u8]) -> ByteBitmap {
|
||||
let mut bm = ByteBitmap::default();
|
||||
for &b in bytes {
|
||||
bm.set(b)
|
||||
}
|
||||
bm
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_search() {
|
||||
assert_eq!(EmptyString {}.find_in(&[1, 2, 3]), Some(0));
|
||||
assert_eq!(EmptyString {}.find_in(&[]), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitmap_search() {
|
||||
assert_eq!(make_bitmap(&[]).find_in(&[1, 2, 3]), None);
|
||||
assert_eq!(make_bitmap(&[]).bitnot().find_in(&[1, 2, 3]), Some(0));
|
||||
assert_eq!(make_bitmap(&[1]).bitnot().find_in(&[1, 2, 3]), Some(1));
|
||||
assert_eq!(make_bitmap(&[2]).bitnot().find_in(&[1, 2, 3]), Some(0));
|
||||
assert_eq!(
|
||||
make_bitmap(&[4, 5, 6, 7]).find_in(&[8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
make_bitmap(&[4, 5, 6, 7]).find_in(&[8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
make_bitmap(&[4, 5, 6, 7])
|
||||
.find_in(&[8, 9, 10, 11, 12, 13, 4, 14, 6, 15, 7, 16, 17, 18, 19, 20]),
|
||||
Some(6)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn literal_search() {
|
||||
assert_eq!([0, 1, 2, 3].find_in(&[4, 5, 6, 7]), None);
|
||||
assert_eq!([0, 1, 2, 3].find_in(&[]), None);
|
||||
}
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
use crate::codepointset::Interval;
|
||||
|
||||
// Character classes like \d or \S.
|
||||
|
||||
/// Construct an interval from an inclusive range of char.
|
||||
const fn r(first: char, last: char) -> Interval {
|
||||
Interval {
|
||||
first: first as u32,
|
||||
last: last as u32,
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct an interval from a single char.
|
||||
const fn r1(c: char) -> Interval {
|
||||
Interval {
|
||||
first: c as u32,
|
||||
last: c as u32,
|
||||
}
|
||||
}
|
||||
|
||||
// Note all of these are sorted.
|
||||
|
||||
/// ES9 21.2.2.6.1.
|
||||
pub const WORD_CHARS: [Interval; 4] = [r('0', '9'), r('A', 'Z'), r1('_'), r('a', 'z')];
|
||||
|
||||
/// ES9 21.2.2.12
|
||||
pub const DIGITS: [Interval; 1] = [r('0', '9')];
|
||||
|
||||
/// [`ES13 12.2 White Space`][spec]
|
||||
///
|
||||
/// [spec]: https://262.ecma-international.org/13.0/#prod-WhiteSpace
|
||||
pub const WHITESPACE: [Interval; 9] = [
|
||||
// U+0009 - Character Tabulation - <TAB>
|
||||
// U+000B - Line Tabulation - <VT>
|
||||
// U+000C - Form Feed (FF) - <FF>
|
||||
r('\u{0009}', '\u{000C}'),
|
||||
// From unicode “Space_Separator” (`Zs`) category:
|
||||
//
|
||||
// U+0020 - Space - <SP>
|
||||
r1('\u{0020}'),
|
||||
// From unicode “Space_Separator” (`Zs`) category:
|
||||
//
|
||||
// U+00A0 - No-Break Space - <NBSP>
|
||||
r1('\u{00A0}'),
|
||||
// From unicode “Space_Separator” (`Zs`) category:
|
||||
//
|
||||
// U+1680 - Ogham Space Mark
|
||||
r1('\u{1680}'),
|
||||
// From unicode “Space_Separator” (`Zs`) category:
|
||||
//
|
||||
// U+2000 - En Quad
|
||||
// U+2001 - Em Quad
|
||||
// U+2002 - En Space
|
||||
// U+2003 - Em Space
|
||||
// U+2004 - Three-Per-Em Space
|
||||
// U+2005 - Four-Per-Em Space
|
||||
// U+2006 - Six-Per-Em Space
|
||||
// U+2007 - Figure Space
|
||||
// U+2008 - Punctuation Space
|
||||
// U+2009 - Thin Space
|
||||
// U+200A - Hair Space
|
||||
r('\u{2000}', '\u{200A}'),
|
||||
// From unicode “Space_Separator” (`Zs`) category:
|
||||
//
|
||||
// U+202F - Narrow No-Break Space - <NNBSP>
|
||||
r1('\u{202F}'),
|
||||
// From unicode “Space_Separator” (`Zs`) category:
|
||||
//
|
||||
// U+205F - Medium Mathematical Space - <MMSP>
|
||||
r1('\u{205F}'),
|
||||
// From unicode “Space_Separator” (`Zs`) category:
|
||||
//
|
||||
// U+3000 - Ideographic Space
|
||||
r1('\u{3000}'),
|
||||
// U+FEFF - ZERO WIDTH NO-BREAK SPACE - <ZWNBSP>
|
||||
r1('\u{FEFF}'),
|
||||
];
|
||||
|
||||
/// ES9 11.3
|
||||
pub const LINE_TERMINATOR: [Interval; 3] =
|
||||
[r1('\u{000A}'), r1('\u{000D}'), r('\u{2028}', '\u{2029}')];
|
||||
+1172
File diff suppressed because it is too large
Load Diff
+498
@@ -0,0 +1,498 @@
|
||||
use crate::util::SliceHelp;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
use core::cmp::{self, Ordering};
|
||||
|
||||
pub type CodePoint = u32;
|
||||
|
||||
/// The maximum (inclusive) code point.
|
||||
pub const CODE_POINT_MAX: CodePoint = 0x10FFFF;
|
||||
|
||||
/// An inclusive range of code points.
|
||||
/// This is more efficient than InclusiveRange because it does not need to carry
|
||||
/// around the `Option<bool>`.
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub struct Interval {
|
||||
pub(crate) first: CodePoint,
|
||||
pub(crate) last: CodePoint,
|
||||
}
|
||||
|
||||
/// A list of sorted, inclusive, non-empty ranges of code points.
|
||||
impl Interval {
|
||||
pub(crate) const fn new(first: CodePoint, last: CodePoint) -> Interval {
|
||||
debug_assert!(first <= last);
|
||||
Interval { first, last }
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn compare(self, cp: u32) -> Ordering {
|
||||
if self.first > cp {
|
||||
Ordering::Greater
|
||||
} else if self.last < cp {
|
||||
Ordering::Less
|
||||
} else {
|
||||
Ordering::Equal
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether self is before rhs.
|
||||
fn is_before(self, other: Interval) -> bool {
|
||||
self.last < other.first
|
||||
}
|
||||
|
||||
/// Return whether self is strictly before rhs.
|
||||
/// "Strictly" here means there is at least one value after the end of self,
|
||||
/// and before the start of rhs. Overlapping *or abutting* intervals are
|
||||
/// not considered strictly before.
|
||||
fn is_strictly_before(self, rhs: Interval) -> bool {
|
||||
self.last + 1 < rhs.first
|
||||
}
|
||||
|
||||
/// Compare two intervals.
|
||||
/// Overlapping *or abutting* intervals are considered equal.
|
||||
fn mergecmp(self, rhs: Interval) -> cmp::Ordering {
|
||||
if self.is_strictly_before(rhs) {
|
||||
Ordering::Less
|
||||
} else if rhs.is_strictly_before(self) {
|
||||
Ordering::Greater
|
||||
} else {
|
||||
Ordering::Equal
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether self is mergeable with rhs.
|
||||
fn mergeable(self, rhs: Interval) -> bool {
|
||||
self.mergecmp(rhs) == Ordering::Equal
|
||||
}
|
||||
|
||||
/// Return whether self contains a code point \p cp.
|
||||
pub fn contains(self, cp: CodePoint) -> bool {
|
||||
self.first <= cp && cp <= self.last
|
||||
}
|
||||
|
||||
/// Return whether self overlaps 'other'.
|
||||
/// Overlaps means that we share at least one code point with 'other'.
|
||||
pub fn overlaps(self, other: Interval) -> bool {
|
||||
!self.is_before(other) && !other.is_before(self)
|
||||
}
|
||||
|
||||
/// Return the interval of codepoints.
|
||||
pub fn codepoints(self) -> core::ops::Range<u32> {
|
||||
debug_assert!(self.last + 1 > self.last, "Overflow");
|
||||
self.first..(self.last + 1)
|
||||
}
|
||||
|
||||
/// Return the number of contained code points.
|
||||
pub fn count_codepoints(self) -> usize {
|
||||
(self.last - self.first + 1) as usize
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn interval_contains(interval: &[Interval], cp: u32) -> bool {
|
||||
interval.binary_search_by(|iv| iv.compare(cp)).is_ok()
|
||||
}
|
||||
|
||||
/// Merge two intervals, which must be overlapping or abutting.
|
||||
fn merge_intervals(x: Interval, y: &Interval) -> Interval {
|
||||
debug_assert!(x.mergeable(*y), "Ranges not mergeable");
|
||||
Interval {
|
||||
first: core::cmp::min(x.first, y.first),
|
||||
last: core::cmp::max(x.last, y.last),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct CodePointSet {
|
||||
ivs: Vec<Interval>,
|
||||
}
|
||||
|
||||
/// A set of code points stored via as disjoint, non-abutting, sorted intervals.
|
||||
impl CodePointSet {
|
||||
pub fn new() -> CodePointSet {
|
||||
CodePointSet { ivs: Vec::new() }
|
||||
}
|
||||
|
||||
pub(crate) fn clear(&mut self) {
|
||||
self.ivs.clear();
|
||||
}
|
||||
|
||||
// Return true if the set is empty.
|
||||
pub(crate) fn is_empty(&self) -> bool {
|
||||
self.ivs.is_empty()
|
||||
}
|
||||
|
||||
// Return true if we contain all code points.
|
||||
pub(crate) fn contains_all_codepoints(&self) -> bool {
|
||||
self.ivs.len() == 1 && self.ivs[0] == Interval::new(0, CODE_POINT_MAX)
|
||||
}
|
||||
|
||||
pub(crate) fn contains(&self, cp: u32) -> bool {
|
||||
interval_contains(&self.ivs, cp)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn assert_is_well_formed(&self) {
|
||||
if cfg!(debug_assertions) {
|
||||
for iv in &self.ivs {
|
||||
debug_assert!(iv.last <= CODE_POINT_MAX);
|
||||
debug_assert!(iv.first <= iv.last);
|
||||
}
|
||||
for w in self.ivs.windows(2) {
|
||||
debug_assert!(w[0].is_strictly_before(w[1]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct from sorted, disjoint intervals. Note these are not allowed to
|
||||
/// even abut.
|
||||
pub fn from_sorted_disjoint_intervals(ivs: Vec<Interval>) -> CodePointSet {
|
||||
let res = CodePointSet { ivs };
|
||||
res.assert_is_well_formed();
|
||||
res
|
||||
}
|
||||
|
||||
/// Add an interval of code points to the set.
|
||||
pub fn add(&mut self, new_iv: Interval) {
|
||||
// Find the mergeable subarray, that is, the range of intervals that intersect
|
||||
// or abut new_iv.
|
||||
let mergeable = self.ivs.equal_range_by(|iv| iv.mergecmp(new_iv));
|
||||
|
||||
// Check our work.
|
||||
if cfg!(debug_assertions) {
|
||||
debug_assert!(new_iv.first <= new_iv.last);
|
||||
for (idx, iv) in self.ivs.iter().enumerate() {
|
||||
if idx < mergeable.start {
|
||||
debug_assert!(iv.is_strictly_before(new_iv));
|
||||
} else if idx >= mergeable.end {
|
||||
debug_assert!(new_iv.is_strictly_before(*iv));
|
||||
} else {
|
||||
debug_assert!(iv.mergeable(new_iv) && new_iv.mergeable(*iv));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge all the overlapping intervals (possibly none), and then replace the
|
||||
// range. Tests show that drain(), which modifies the vector, is not effectively
|
||||
// optimized, so try to avoid it in the cases of a new entry or replacing an existing
|
||||
// entry.
|
||||
match mergeable.end - mergeable.start {
|
||||
0 => {
|
||||
// New entry.
|
||||
self.ivs.insert(mergeable.start, new_iv);
|
||||
}
|
||||
1 => {
|
||||
// Replace a single entry.
|
||||
let entry = &mut self.ivs[mergeable.start];
|
||||
*entry = Interval {
|
||||
first: cmp::min(entry.first, new_iv.first),
|
||||
last: cmp::max(entry.last, new_iv.last),
|
||||
};
|
||||
}
|
||||
_ => {
|
||||
// Replace range of entries.
|
||||
let merged_iv: Interval = self.ivs[mergeable.clone()]
|
||||
.iter()
|
||||
.fold(new_iv, merge_intervals);
|
||||
self.ivs[mergeable.start] = merged_iv;
|
||||
self.ivs.drain(mergeable.start + 1..mergeable.end);
|
||||
}
|
||||
}
|
||||
self.assert_is_well_formed();
|
||||
}
|
||||
|
||||
/// Add a single code point to the set.
|
||||
#[inline]
|
||||
pub fn add_one(&mut self, cp: CodePoint) {
|
||||
self.add(Interval {
|
||||
first: cp,
|
||||
last: cp,
|
||||
})
|
||||
}
|
||||
|
||||
/// Add another code point set.
|
||||
pub fn add_set(&mut self, mut rhs: CodePointSet) {
|
||||
// Prefer to add to the set with more intervals.
|
||||
if self.ivs.len() < rhs.ivs.len() {
|
||||
core::mem::swap(self, &mut rhs);
|
||||
}
|
||||
for iv in rhs.intervals() {
|
||||
self.add(*iv)
|
||||
}
|
||||
}
|
||||
|
||||
/// \return the intervals
|
||||
pub fn intervals(&self) -> &[Interval] {
|
||||
self.ivs.as_slice()
|
||||
}
|
||||
|
||||
/// \return the number of intervals that would be produced by inverting.
|
||||
pub fn inverted_interval_count(&self) -> usize {
|
||||
let mut result = 0;
|
||||
let mut start: CodePoint = 0;
|
||||
for iv in &self.ivs {
|
||||
if start < iv.first {
|
||||
result += 1;
|
||||
}
|
||||
start = iv.last + 1;
|
||||
}
|
||||
if start <= CODE_POINT_MAX {
|
||||
result += 1;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// \return an inverted set: a set containing every code point NOT in the
|
||||
/// receiver.
|
||||
pub fn inverted(&self) -> CodePointSet {
|
||||
// The intervals we collect.
|
||||
let mut inverted_ivs = Vec::new();
|
||||
|
||||
// The first code point *not* in the previous interval.
|
||||
let mut start: CodePoint = 0;
|
||||
for iv in &self.ivs {
|
||||
if start < iv.first {
|
||||
inverted_ivs.push(Interval {
|
||||
first: start,
|
||||
last: iv.first - 1,
|
||||
})
|
||||
}
|
||||
start = iv.last + 1;
|
||||
}
|
||||
if start <= CODE_POINT_MAX {
|
||||
inverted_ivs.push(Interval {
|
||||
first: start,
|
||||
last: CODE_POINT_MAX,
|
||||
})
|
||||
}
|
||||
CodePointSet::from_sorted_disjoint_intervals(inverted_ivs)
|
||||
}
|
||||
|
||||
/// Remove the the given intervals from the set.
|
||||
///
|
||||
/// Invariants: The intervals must be sorted and disjoint.
|
||||
pub(crate) fn remove(&mut self, intervals: &[Interval]) {
|
||||
let mut result = Vec::new();
|
||||
let mut remove_iter = intervals.iter().peekable();
|
||||
let mut current_remove = remove_iter.next();
|
||||
|
||||
for iv in &mut self.ivs {
|
||||
while let Some(remove_iv) = current_remove {
|
||||
if remove_iv.last < iv.first {
|
||||
current_remove = remove_iter.next();
|
||||
} else if remove_iv.first > iv.last {
|
||||
result.push(*iv);
|
||||
break;
|
||||
} else {
|
||||
if remove_iv.first > iv.first {
|
||||
result.push(Interval {
|
||||
first: iv.first,
|
||||
last: remove_iv.first - 1,
|
||||
});
|
||||
}
|
||||
if remove_iv.last < iv.last {
|
||||
iv.first = remove_iv.last + 1;
|
||||
current_remove = remove_iter.next();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if current_remove.is_none() {
|
||||
result.push(*iv);
|
||||
}
|
||||
}
|
||||
|
||||
self.ivs = result;
|
||||
}
|
||||
|
||||
/// Intersect the set with the given intervals.
|
||||
pub(crate) fn intersect(&mut self, intervals: &[Interval]) {
|
||||
let mut new_ivs = Vec::new();
|
||||
for iv in intervals {
|
||||
for self_iv in self.intervals() {
|
||||
if iv.overlaps(*self_iv) {
|
||||
new_ivs.push(Interval {
|
||||
first: cmp::max(iv.first, self_iv.first),
|
||||
last: cmp::min(iv.last, self_iv.last),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
self.ivs = new_ivs;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn iv(first: u32, last: u32) -> Interval {
|
||||
Interval { first, last }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_before() {
|
||||
let a = iv(0, 9);
|
||||
let b = iv(10, 19);
|
||||
assert!(a.is_before(b));
|
||||
assert!(!b.is_before(a));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_strictly_before() {
|
||||
let a = iv(0, 9);
|
||||
let b = iv(10, 19);
|
||||
let c = iv(11, 19);
|
||||
assert!(!a.is_strictly_before(b));
|
||||
assert!(a.is_strictly_before(c));
|
||||
assert!(!b.is_strictly_before(a));
|
||||
assert!(!b.is_strictly_before(c));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mergecmp() {
|
||||
let a = iv(0, 9);
|
||||
let b = iv(10, 19);
|
||||
let c = iv(9, 18);
|
||||
assert_eq!(a.mergecmp(b), Ordering::Equal);
|
||||
assert_eq!(b.mergecmp(a), Ordering::Equal);
|
||||
assert_eq!(a.mergecmp(c), Ordering::Equal);
|
||||
assert_eq!(c.mergecmp(a), Ordering::Equal);
|
||||
|
||||
let d = iv(11, 19);
|
||||
assert_eq!(a.mergecmp(d), Ordering::Less);
|
||||
assert_eq!(d.mergecmp(a), Ordering::Greater);
|
||||
assert_eq!(b.mergecmp(d), Ordering::Equal);
|
||||
assert_eq!(d.mergecmp(b), Ordering::Equal);
|
||||
assert_eq!(c.mergecmp(d), Ordering::Equal);
|
||||
assert_eq!(d.mergecmp(c), Ordering::Equal);
|
||||
|
||||
let e = iv(100, 109);
|
||||
assert_eq!(a.mergecmp(e), Ordering::Less);
|
||||
assert_eq!(e.mergecmp(a), Ordering::Greater);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mergeable() {
|
||||
let a = iv(0, 9);
|
||||
let b = iv(9, 19);
|
||||
assert!(a.mergeable(a));
|
||||
assert!(a.mergeable(b));
|
||||
assert!(b.mergeable(b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_contains() {
|
||||
let a = iv(0, 9);
|
||||
assert!(a.contains(0));
|
||||
assert!(a.contains(9));
|
||||
assert!(!a.contains(10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_overlaps() {
|
||||
let a = iv(0, 9);
|
||||
let b = iv(5, 14);
|
||||
let c = iv(10, 19);
|
||||
assert!(a.overlaps(b));
|
||||
assert!(!a.overlaps(c));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_codepoints() {
|
||||
let a = iv(0, 9);
|
||||
assert_eq!(a.codepoints(), 0..10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_count_codepoints() {
|
||||
assert_eq!(iv(0, 9).count_codepoints(), 10);
|
||||
assert_eq!(iv(0, 0).count_codepoints(), 1);
|
||||
assert_eq!(
|
||||
iv(0, CODE_POINT_MAX).count_codepoints(),
|
||||
(CODE_POINT_MAX + 1) as usize
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add() {
|
||||
let mut set = CodePointSet::new();
|
||||
set.add(iv(10, 20));
|
||||
set.add(iv(30, 40));
|
||||
set.add(iv(15, 35));
|
||||
assert_eq!(set.intervals(), &[iv(10, 40)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_one() {
|
||||
let mut set = CodePointSet::new();
|
||||
set.add_one(10);
|
||||
set.add_one(20);
|
||||
set.add_one(15);
|
||||
assert_eq!(set.intervals(), &[iv(10, 10), iv(15, 15), iv(20, 20)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_set() {
|
||||
let mut set1 = CodePointSet::new();
|
||||
set1.add(iv(10, 20));
|
||||
set1.add(iv(30, 40));
|
||||
let mut set2 = CodePointSet::new();
|
||||
set2.add(iv(15, 25));
|
||||
set2.add(iv(35, 45));
|
||||
set1.add_set(set2);
|
||||
assert_eq!(set1.intervals(), &[iv(10, 25), iv(30, 45)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inverted() {
|
||||
let mut set = CodePointSet::new();
|
||||
set.add(iv(10, 20));
|
||||
set.add(iv(30, 40));
|
||||
let inverted_set = set.inverted();
|
||||
assert_eq!(
|
||||
inverted_set.intervals(),
|
||||
&[iv(0, 9), iv(21, 29), iv(41, CODE_POINT_MAX)]
|
||||
);
|
||||
let set_again = inverted_set.inverted();
|
||||
assert_eq!(set_again.intervals(), set.intervals());
|
||||
|
||||
assert_eq!(
|
||||
set.inverted_interval_count(),
|
||||
inverted_set.intervals().len()
|
||||
);
|
||||
assert_eq!(
|
||||
inverted_set.inverted_interval_count(),
|
||||
set.intervals().len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_adds_torture() {
|
||||
let mut set = CodePointSet::new();
|
||||
set.add(iv(1, 3));
|
||||
assert_eq!(&set.intervals(), &[iv(1, 3)]);
|
||||
set.add(iv(0, 0));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 3)]);
|
||||
set.add(iv(3, 5));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 5)]);
|
||||
set.add(iv(6, 10));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 10)]);
|
||||
set.add(iv(15, 15));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 10), iv(15, 15)]);
|
||||
set.add(iv(12, 14));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 10), iv(12, 15)]);
|
||||
set.add(iv(16, 20));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 10), iv(12, 20)]);
|
||||
set.add(iv(21, 22));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 10), iv(12, 22)]);
|
||||
set.add(iv(23, 23));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 10), iv(12, 23)]);
|
||||
set.add(iv(100, 200));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 10), iv(12, 23), iv(100, 200)]);
|
||||
set.add(iv(201, 250));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 10), iv(12, 23), iv(100, 250)]);
|
||||
set.add(iv(0, 0x10ffff));
|
||||
assert_eq!(&set.intervals(), &[iv(0, 0x10ffff)]);
|
||||
}
|
||||
}
|
||||
Vendored
+77
@@ -0,0 +1,77 @@
|
||||
use crate::indexing::InputIndexer;
|
||||
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct Forward;
|
||||
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct Backward;
|
||||
|
||||
pub trait Direction: core::fmt::Debug + Copy + Clone {
|
||||
const FORWARD: bool;
|
||||
fn new() -> Self;
|
||||
}
|
||||
|
||||
impl Direction for Forward {
|
||||
const FORWARD: bool = true;
|
||||
#[inline(always)]
|
||||
fn new() -> Self {
|
||||
Forward {}
|
||||
}
|
||||
}
|
||||
|
||||
impl Direction for Backward {
|
||||
const FORWARD: bool = false;
|
||||
#[inline(always)]
|
||||
fn new() -> Self {
|
||||
Backward {}
|
||||
}
|
||||
}
|
||||
|
||||
/// \return whether we match some literal bytes.
|
||||
/// If so, update the position. If not, the position is unspecified.
|
||||
#[inline(always)]
|
||||
pub fn try_match_lit<const N: usize, Input: InputIndexer, Dir: Direction>(
|
||||
input: &Input,
|
||||
dir: Dir,
|
||||
pos: &mut Input::Position,
|
||||
bytes: &[u8; N],
|
||||
) -> bool {
|
||||
input.match_bytes(dir, pos, bytes)
|
||||
}
|
||||
|
||||
/// \return the next character, updating the position.
|
||||
#[inline(always)]
|
||||
pub fn next<Input: InputIndexer, Dir: Direction>(
|
||||
input: &Input,
|
||||
_dir: Dir,
|
||||
pos: &mut Input::Position,
|
||||
) -> Option<Input::Element> {
|
||||
if Dir::FORWARD {
|
||||
input.next_right(pos)
|
||||
} else {
|
||||
input.next_left(pos)
|
||||
}
|
||||
}
|
||||
|
||||
/// \return the next *byte*, or None if at the end, updating the position.
|
||||
/// Note this may break UTF8 sequences.
|
||||
#[inline(always)]
|
||||
pub fn next_byte<Input: InputIndexer, Dir: Direction>(
|
||||
input: &Input,
|
||||
_dir: Dir,
|
||||
pos: &mut Input::Position,
|
||||
) -> Option<u8> {
|
||||
assert!(
|
||||
Input::CODE_UNITS_ARE_BYTES,
|
||||
"Not implemented for non-byte input"
|
||||
);
|
||||
let res;
|
||||
if Dir::FORWARD {
|
||||
res = input.peek_byte_right(*pos);
|
||||
*pos += if res.is_some() { 1 } else { 0 };
|
||||
} else {
|
||||
res = input.peek_byte_left(*pos);
|
||||
*pos -= if res.is_some() { 1 } else { 0 };
|
||||
}
|
||||
res
|
||||
}
|
||||
Vendored
+410
@@ -0,0 +1,410 @@
|
||||
//! Regex compiler back-end: transforms IR into a CompiledRegex
|
||||
|
||||
use crate::bytesearch::{AsciiBitmap, ByteArraySet};
|
||||
use crate::insn::{CompiledRegex, Insn, LoopFields, MAX_BYTE_SEQ_LENGTH, MAX_CHAR_SET_LENGTH};
|
||||
use crate::ir;
|
||||
use crate::ir::Node;
|
||||
use crate::startpredicate;
|
||||
use crate::types::{BracketContents, CaptureGroupID, LoopID};
|
||||
use crate::unicode;
|
||||
use core::convert::TryInto;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use {alloc::vec::Vec, hashbrown::HashMap};
|
||||
|
||||
/// \return an anchor instruction for a given IR anchor.
|
||||
fn make_anchor(anchor_type: ir::AnchorType, multiline: bool) -> Insn {
|
||||
match anchor_type {
|
||||
ir::AnchorType::StartOfLine => Insn::StartOfLine { multiline },
|
||||
ir::AnchorType::EndOfLine => Insn::EndOfLine { multiline },
|
||||
}
|
||||
}
|
||||
|
||||
/// Weirdly placed optimization.
|
||||
/// If the given bracket can be represented as ASCII contents, return the
|
||||
/// bitmap. Otherwise nothing.
|
||||
fn bracket_as_ascii(bc: &BracketContents) -> Option<AsciiBitmap> {
|
||||
let mut result = AsciiBitmap::default();
|
||||
// We just assume that inverted brackets contain non-ASCII characters.
|
||||
if bc.invert {
|
||||
return None;
|
||||
}
|
||||
for r in bc.cps.intervals() {
|
||||
debug_assert!(r.first <= r.last);
|
||||
if r.last >= 128 {
|
||||
return None;
|
||||
}
|
||||
for bit in r.first..=r.last {
|
||||
result.set(bit as u8)
|
||||
}
|
||||
}
|
||||
Some(result)
|
||||
}
|
||||
|
||||
/// Type which wraps up the context needed to emit a CompiledRegex.
|
||||
struct Emitter {
|
||||
result: CompiledRegex,
|
||||
|
||||
// Number of loops seen so far.
|
||||
next_loop_id: LoopID,
|
||||
|
||||
// List of group names, in order, with empties for unnamed groups.
|
||||
group_names: Vec<Box<str>>,
|
||||
|
||||
// Whether the current node is within a lookbehind.
|
||||
in_lookbehind: bool,
|
||||
}
|
||||
|
||||
impl Emitter {
|
||||
/// Emit a ByteSet instruction.
|
||||
/// We awkwardly optimize it like so.
|
||||
fn emit_byte_set_insn(&mut self, bytes: &[u8]) {
|
||||
let insn = match bytes.len() {
|
||||
0 => Insn::JustFail,
|
||||
1 => Insn::ByteSeq1(bytes.try_into().unwrap()),
|
||||
2 => Insn::ByteSet2(ByteArraySet(bytes.try_into().unwrap())),
|
||||
3 => Insn::ByteSet3(ByteArraySet(bytes.try_into().unwrap())),
|
||||
4 => Insn::ByteSet4(ByteArraySet(bytes.try_into().unwrap())),
|
||||
_ => panic!("Byte set is too long"),
|
||||
};
|
||||
self.emit_insn(insn);
|
||||
}
|
||||
|
||||
// Emit a nonempty byte sequence instruction. The sequence must be at most MAX_BYTE_SEQ_LENGTH bytes.
|
||||
fn emit_byte_sequence_insn(&mut self, seq: &[u8]) {
|
||||
const {
|
||||
assert!(
|
||||
MAX_BYTE_SEQ_LENGTH == 16,
|
||||
"Need to update our emitting logic"
|
||||
);
|
||||
};
|
||||
let insn = match seq.len() {
|
||||
1 => Insn::ByteSeq1(seq.try_into().unwrap()),
|
||||
2 => Insn::ByteSeq2(seq.try_into().unwrap()),
|
||||
3 => Insn::ByteSeq3(seq.try_into().unwrap()),
|
||||
4 => Insn::ByteSeq4(seq.try_into().unwrap()),
|
||||
5 => Insn::ByteSeq5(seq.try_into().unwrap()),
|
||||
6 => Insn::ByteSeq6(seq.try_into().unwrap()),
|
||||
7 => Insn::ByteSeq7(seq.try_into().unwrap()),
|
||||
8 => Insn::ByteSeq8(seq.try_into().unwrap()),
|
||||
9 => Insn::ByteSeq9(seq.try_into().unwrap()),
|
||||
10 => Insn::ByteSeq10(seq.try_into().unwrap()),
|
||||
11 => Insn::ByteSeq11(seq.try_into().unwrap()),
|
||||
12 => Insn::ByteSeq12(seq.try_into().unwrap()),
|
||||
13 => Insn::ByteSeq13(seq.try_into().unwrap()),
|
||||
14 => Insn::ByteSeq14(seq.try_into().unwrap()),
|
||||
15 => Insn::ByteSeq15(seq.try_into().unwrap()),
|
||||
16 => Insn::ByteSeq16(seq.try_into().unwrap()),
|
||||
_ => panic!("Unexpected chunk size"),
|
||||
};
|
||||
self.emit_insn(insn);
|
||||
}
|
||||
|
||||
/// Emit an instruction.
|
||||
/// Return the "instruction" as an index.
|
||||
fn emit_insn(&mut self, insn: Insn) {
|
||||
self.result.insns.push(insn);
|
||||
}
|
||||
|
||||
/// Get an instruction at a given index.
|
||||
fn get_insn(&mut self, idx: u32) -> &mut Insn {
|
||||
&mut self.result.insns[idx as usize]
|
||||
}
|
||||
|
||||
/// \return the offset of the next instruction emitted.
|
||||
fn next_offset(&self) -> u32 {
|
||||
self.result.insns.len() as u32
|
||||
}
|
||||
|
||||
fn emit_insn_offset(&mut self, insn: Insn) -> u32 {
|
||||
let ret = self.next_offset();
|
||||
self.emit_insn(insn);
|
||||
ret
|
||||
}
|
||||
|
||||
/// Emit instructions corresponding to a given node.
|
||||
fn emit_node(&mut self, node: &Node) {
|
||||
enum Emitter<'a> {
|
||||
Node(&'a Node),
|
||||
NodeLoopFinish {
|
||||
loop_instruction_index: u32,
|
||||
},
|
||||
NodeLookaroundAssertionFinish {
|
||||
lookaround_instruction_index: u32,
|
||||
prev_in_lookbehind: bool,
|
||||
},
|
||||
NodeAltMiddle {
|
||||
alt_instruction_index: u32,
|
||||
right_node: &'a Node,
|
||||
},
|
||||
NodeAltFinish {
|
||||
alt_instruction_index: u32,
|
||||
jump_instruction_index: u32,
|
||||
right_branch_index: u32,
|
||||
},
|
||||
EndCaptureGroup {
|
||||
group: CaptureGroupID,
|
||||
},
|
||||
}
|
||||
|
||||
let mut stack = vec![Emitter::Node(node)];
|
||||
|
||||
while let Some(inst) = stack.pop() {
|
||||
match inst {
|
||||
Emitter::NodeLoopFinish {
|
||||
loop_instruction_index,
|
||||
} => {
|
||||
self.emit_insn(Insn::LoopAgain {
|
||||
begin: loop_instruction_index,
|
||||
});
|
||||
// Fix up our loop exit.
|
||||
let exit = self.next_offset();
|
||||
match self.get_insn(loop_instruction_index) {
|
||||
Insn::EnterLoop(fields) => fields.exit = exit,
|
||||
_ => panic!("Should be an EnterLoop instruction"),
|
||||
}
|
||||
}
|
||||
Emitter::NodeLookaroundAssertionFinish {
|
||||
lookaround_instruction_index,
|
||||
prev_in_lookbehind,
|
||||
} => {
|
||||
self.emit_insn(Insn::Goal);
|
||||
// Fix up the continuation.
|
||||
let next_insn = self.next_offset();
|
||||
match self.get_insn(lookaround_instruction_index) {
|
||||
Insn::Lookbehind { continuation, .. } => *continuation = next_insn,
|
||||
Insn::Lookahead { continuation, .. } => *continuation = next_insn,
|
||||
_ => panic!("Should be a Lookaround instruction"),
|
||||
}
|
||||
self.in_lookbehind = prev_in_lookbehind;
|
||||
}
|
||||
Emitter::NodeAltMiddle {
|
||||
alt_instruction_index,
|
||||
right_node,
|
||||
} => {
|
||||
let jump_insn = self.emit_insn_offset(Insn::Jump { target: 0 });
|
||||
let right_branch = self.next_offset();
|
||||
stack.push(Emitter::NodeAltFinish {
|
||||
alt_instruction_index,
|
||||
jump_instruction_index: jump_insn,
|
||||
right_branch_index: right_branch,
|
||||
});
|
||||
stack.push(Emitter::Node(right_node));
|
||||
}
|
||||
Emitter::NodeAltFinish {
|
||||
alt_instruction_index,
|
||||
jump_instruction_index,
|
||||
right_branch_index,
|
||||
} => {
|
||||
let exit = self.next_offset();
|
||||
// Fix up our jump targets.
|
||||
match self.get_insn(alt_instruction_index) {
|
||||
Insn::Alt { secondary } => *secondary = right_branch_index,
|
||||
_ => panic!("Should be an Alt instruction"),
|
||||
}
|
||||
match self.get_insn(jump_instruction_index) {
|
||||
Insn::Jump { target } => *target = exit,
|
||||
_ => panic!("Should be a Jump instruction"),
|
||||
}
|
||||
}
|
||||
Emitter::EndCaptureGroup { group } => self.emit_insn(Insn::EndCaptureGroup(group)),
|
||||
Emitter::Node(node) => match node {
|
||||
Node::Empty => {}
|
||||
Node::Goal => self.emit_insn(Insn::Goal),
|
||||
Node::Char { c, icase } => {
|
||||
let c = *c;
|
||||
if !*icase {
|
||||
self.emit_insn(Insn::Char(c))
|
||||
} else {
|
||||
core::debug_assert!(
|
||||
unicode::fold_code_point(c, self.result.flags.unicode) == c,
|
||||
"Char {:x} should be folded",
|
||||
c
|
||||
);
|
||||
self.emit_insn(Insn::CharICase(c))
|
||||
}
|
||||
}
|
||||
Node::Cat(children) => {
|
||||
for nn in children.iter().rev() {
|
||||
stack.push(Emitter::Node(nn));
|
||||
}
|
||||
}
|
||||
Node::Alt(left, right) => {
|
||||
// Alternation is followed by the primary branch and has a jump to secondary
|
||||
// branch. After primary branch, jump to the continuation.
|
||||
let alt_insn = self.emit_insn_offset(Insn::Alt { secondary: 0 });
|
||||
stack.push(Emitter::NodeAltMiddle {
|
||||
alt_instruction_index: alt_insn,
|
||||
right_node: right,
|
||||
});
|
||||
stack.push(Emitter::Node(left));
|
||||
}
|
||||
Node::Bracket(contents) => {
|
||||
if let Some(ascii_contents) = bracket_as_ascii(contents) {
|
||||
self.emit_insn(Insn::AsciiBracket(ascii_contents))
|
||||
} else {
|
||||
let idx = self.result.brackets.len();
|
||||
self.result.brackets.push(contents.clone());
|
||||
self.emit_insn(Insn::Bracket(idx))
|
||||
}
|
||||
}
|
||||
Node::MatchAny => self.emit_insn(Insn::MatchAny),
|
||||
Node::MatchAnyExceptLineTerminator => {
|
||||
self.emit_insn(Insn::MatchAnyExceptLineTerminator)
|
||||
}
|
||||
Node::Anchor {
|
||||
anchor_type,
|
||||
multiline,
|
||||
} => self.emit_insn(make_anchor(*anchor_type, *multiline)),
|
||||
Node::Loop {
|
||||
loopee,
|
||||
quant,
|
||||
enclosed_groups,
|
||||
} => {
|
||||
let loop_id = self.next_loop_id;
|
||||
self.next_loop_id += 1;
|
||||
let loop_insn = self.emit_insn_offset(Insn::EnterLoop(LoopFields {
|
||||
loop_id,
|
||||
min_iters: quant.min,
|
||||
// If the loop is unbounded, just emit usize::MAX,
|
||||
// as we cannot match more characters than that.
|
||||
max_iters: quant.max.unwrap_or(usize::MAX),
|
||||
greedy: quant.greedy,
|
||||
exit: 0,
|
||||
}));
|
||||
self.result.loops += 1;
|
||||
// Emit a sequence of ResetCaptureGroup for any contained groups.
|
||||
for gid in enclosed_groups.start..enclosed_groups.end {
|
||||
self.emit_insn(Insn::ResetCaptureGroup(gid))
|
||||
}
|
||||
stack.push(Emitter::NodeLoopFinish {
|
||||
loop_instruction_index: loop_insn,
|
||||
});
|
||||
stack.push(Emitter::Node(loopee));
|
||||
}
|
||||
Node::Loop1CharBody { loopee, quant } => {
|
||||
self.emit_insn(Insn::Loop1CharBody {
|
||||
min_iters: quant.min,
|
||||
max_iters: quant.max.unwrap_or(usize::MAX),
|
||||
greedy: quant.greedy,
|
||||
});
|
||||
stack.push(Emitter::Node(loopee));
|
||||
}
|
||||
Node::CaptureGroup { id, contents, name } => {
|
||||
let group = *id;
|
||||
self.result.groups += 1;
|
||||
self.group_names.push(name.as_deref().unwrap_or("").into());
|
||||
self.emit_insn(Insn::BeginCaptureGroup(group));
|
||||
stack.push(Emitter::EndCaptureGroup { group });
|
||||
stack.push(Emitter::Node(contents));
|
||||
}
|
||||
Node::LookaroundAssertion {
|
||||
negate,
|
||||
backwards,
|
||||
start_group,
|
||||
end_group,
|
||||
contents,
|
||||
} => {
|
||||
let lookaround = if *backwards {
|
||||
self.emit_insn_offset(Insn::Lookbehind {
|
||||
negate: *negate,
|
||||
start_group: *start_group,
|
||||
end_group: *end_group,
|
||||
continuation: 0,
|
||||
})
|
||||
} else {
|
||||
self.emit_insn_offset(Insn::Lookahead {
|
||||
negate: *negate,
|
||||
start_group: *start_group,
|
||||
end_group: *end_group,
|
||||
continuation: 0,
|
||||
})
|
||||
};
|
||||
let prev_in_lookbehind = self.in_lookbehind;
|
||||
self.in_lookbehind = *backwards;
|
||||
stack.push(Emitter::NodeLookaroundAssertionFinish {
|
||||
lookaround_instruction_index: lookaround,
|
||||
prev_in_lookbehind,
|
||||
});
|
||||
stack.push(Emitter::Node(contents));
|
||||
}
|
||||
&Node::WordBoundary { invert } => {
|
||||
if self.result.flags.unicode && self.result.flags.icase {
|
||||
self.emit_insn(Insn::WordBoundaryUnicodeICase { invert })
|
||||
} else {
|
||||
self.emit_insn(Insn::WordBoundary { invert })
|
||||
}
|
||||
}
|
||||
&Node::BackRef(group) => {
|
||||
debug_assert!(group >= 1, "Group should not be zero");
|
||||
// -1 because \1 matches the first capture group, which has index 0.
|
||||
self.emit_insn(Insn::BackRef(group - 1))
|
||||
}
|
||||
|
||||
Node::ByteSet(bytes) => self.emit_byte_set_insn(bytes),
|
||||
|
||||
Node::CharSet(chars) => {
|
||||
debug_assert!(chars.len() <= MAX_CHAR_SET_LENGTH);
|
||||
if chars.is_empty() {
|
||||
self.emit_insn(Insn::JustFail);
|
||||
} else {
|
||||
let mut arr = [chars[0]; MAX_CHAR_SET_LENGTH];
|
||||
arr[..chars.len()].copy_from_slice(chars.as_slice());
|
||||
self.emit_insn(Insn::CharSet(arr))
|
||||
}
|
||||
}
|
||||
|
||||
Node::ByteSequence(bytes) => {
|
||||
// In a lookbehind, instruction order is reversed, but ByteSequence contents are not,
|
||||
// to take advantage of optimized memcmp. For example, `(?<=abcd)` may be emitted
|
||||
// as "d", "c", "b", "a" char matches (traversing input from right to left), or
|
||||
// optimized to a single forwards ByteSequence("abcd"), which matches left-to-right.
|
||||
// Therefore if we split a byte sequence into multiple instructions,
|
||||
// we need to emit them in reverse order iff in a lookbehind.
|
||||
let chunks = bytes.as_slice().chunks(MAX_BYTE_SEQ_LENGTH);
|
||||
if self.in_lookbehind {
|
||||
for chunk in chunks.rev() {
|
||||
self.emit_byte_sequence_insn(chunk);
|
||||
}
|
||||
} else {
|
||||
for chunk in chunks {
|
||||
self.emit_byte_sequence_insn(chunk);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compile the given IR to a CompiledRegex.
|
||||
pub fn emit(n: &ir::Regex) -> CompiledRegex {
|
||||
let mut emitter = Emitter {
|
||||
next_loop_id: 0,
|
||||
group_names: Vec::new(),
|
||||
in_lookbehind: false,
|
||||
result: CompiledRegex {
|
||||
insns: Vec::new(),
|
||||
brackets: Vec::new(),
|
||||
loops: 0,
|
||||
groups: 0,
|
||||
group_names: Box::new([]),
|
||||
flags: n.flags,
|
||||
start_pred: startpredicate::predicate_for_re(n),
|
||||
},
|
||||
};
|
||||
emitter.emit_node(&n.node);
|
||||
let mut result = emitter.result;
|
||||
// Populate group names, unless all are empty.
|
||||
debug_assert!(
|
||||
result.group_names.is_empty(),
|
||||
"Group names should not be set"
|
||||
);
|
||||
if emitter.group_names.iter().any(|s| !s.is_empty()) {
|
||||
result.group_names = emitter.group_names.into_boxed_slice();
|
||||
}
|
||||
debug_assert!(
|
||||
result.group_names.is_empty() || result.group_names.len() == result.groups as usize
|
||||
);
|
||||
result
|
||||
}
|
||||
Vendored
+56
@@ -0,0 +1,56 @@
|
||||
//! Execution engine bits.
|
||||
|
||||
use crate::api::Match;
|
||||
use crate::insn::CompiledRegex;
|
||||
use crate::position::PositionType;
|
||||
|
||||
/// A trait for finding the next match in a regex.
|
||||
/// This is broken out from Executor to avoid needing to thread lifetimes
|
||||
/// around.
|
||||
pub trait MatchProducer: core::fmt::Debug {
|
||||
/// The position type of our indexer.
|
||||
type Position: PositionType;
|
||||
|
||||
/// \return an initial position for the given start offset.
|
||||
fn initial_position(&self, offset: usize) -> Option<Self::Position>;
|
||||
|
||||
/// Attempt to match at the given location.
|
||||
/// \return either the Match and the position to start looking for the next
|
||||
/// match, or None on failure.
|
||||
fn next_match(
|
||||
&mut self,
|
||||
pos: Self::Position,
|
||||
next_start: &mut Option<Self::Position>,
|
||||
) -> Option<Match>;
|
||||
}
|
||||
|
||||
/// A trait for executing a regex.
|
||||
pub trait Executor<'r, 't>: MatchProducer {
|
||||
/// The ASCII variant.
|
||||
type AsAscii: Executor<'r, 't>;
|
||||
|
||||
/// Construct a new Executor.
|
||||
fn new(re: &'r CompiledRegex, text: &'t str) -> Self;
|
||||
}
|
||||
|
||||
/// A struct which enables iteration over matches.
|
||||
#[derive(Debug)]
|
||||
pub struct Matches<Producer: MatchProducer> {
|
||||
mp: Producer,
|
||||
position: Option<Producer::Position>,
|
||||
}
|
||||
|
||||
impl<Producer: MatchProducer> Matches<Producer> {
|
||||
pub fn new(mp: Producer, start: usize) -> Self {
|
||||
let position = mp.initial_position(start);
|
||||
Matches { mp, position }
|
||||
}
|
||||
}
|
||||
|
||||
impl<Producer: MatchProducer> Iterator for Matches<Producer> {
|
||||
type Item = Match;
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let pos = self.position?;
|
||||
self.mp.next_match(pos, &mut self.position)
|
||||
}
|
||||
}
|
||||
Vendored
+1292
File diff suppressed because it is too large
Load Diff
Vendored
+220
@@ -0,0 +1,220 @@
|
||||
//! Bytecode instructions for a compiled regex
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use {
|
||||
alloc::{string::String, vec::Vec},
|
||||
hashbrown::HashMap,
|
||||
};
|
||||
|
||||
use crate::api;
|
||||
use crate::bytesearch::{AsciiBitmap, ByteArraySet, ByteBitmap};
|
||||
use crate::types::{BracketContents, CaptureGroupID, LoopID};
|
||||
extern crate memchr;
|
||||
use memchr::memmem;
|
||||
|
||||
type JumpTarget = u32;
|
||||
|
||||
/// The maximum size of a byte sequence instruction.
|
||||
pub const MAX_BYTE_SEQ_LENGTH: usize = 16;
|
||||
|
||||
/// The maximum size of an array-type-byteset instruction.
|
||||
pub const MAX_BYTE_SET_LENGTH: usize = 4;
|
||||
|
||||
/// The maximum size of an array-type-charset instruction.
|
||||
/// This also happens to be the maximum number of characters in case-insensitive
|
||||
/// equivalence classes.
|
||||
pub const MAX_CHAR_SET_LENGTH: usize = 4;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LoopFields {
|
||||
pub loop_id: LoopID,
|
||||
pub min_iters: usize,
|
||||
pub max_iters: usize,
|
||||
pub greedy: bool,
|
||||
pub exit: JumpTarget,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
/// The list of bytecode instructions.
|
||||
pub enum Insn {
|
||||
/// The match was successful.
|
||||
Goal,
|
||||
|
||||
/// Match a single char.
|
||||
Char(u32),
|
||||
|
||||
/// Match a single char, case-insensitive.
|
||||
CharICase(u32),
|
||||
|
||||
/// Match the start of a line (if multiline); emitted by '^'
|
||||
StartOfLine {
|
||||
multiline: bool,
|
||||
},
|
||||
|
||||
/// Match the end of a line; emitted by '$'
|
||||
EndOfLine {
|
||||
multiline: bool,
|
||||
},
|
||||
|
||||
/// Match any character except a line terminator; emitted by '.' only when
|
||||
/// the dot_all flag is set to true.
|
||||
MatchAny,
|
||||
|
||||
/// Match any character except a line terminator; emitted by '.'
|
||||
MatchAnyExceptLineTerminator,
|
||||
|
||||
/// Enter a loop from "outside".
|
||||
EnterLoop(LoopFields),
|
||||
|
||||
/// Re-enter a loop.
|
||||
LoopAgain {
|
||||
begin: JumpTarget,
|
||||
},
|
||||
|
||||
/// The next instruction is a "1Char" instruction which always matches one
|
||||
/// character. Attempt to match it [min, max] times.
|
||||
Loop1CharBody {
|
||||
min_iters: usize,
|
||||
max_iters: usize,
|
||||
greedy: bool,
|
||||
},
|
||||
|
||||
/// Set the IP to a new value.
|
||||
Jump {
|
||||
target: JumpTarget,
|
||||
},
|
||||
|
||||
/// The next instruction is the primary branch.
|
||||
/// If it fails to match, jump to secondary.
|
||||
Alt {
|
||||
secondary: JumpTarget,
|
||||
},
|
||||
|
||||
/// Enter a capture group.
|
||||
BeginCaptureGroup(CaptureGroupID),
|
||||
|
||||
/// Exit a capture group.
|
||||
EndCaptureGroup(CaptureGroupID),
|
||||
|
||||
/// Clear a capture group.
|
||||
ResetCaptureGroup(CaptureGroupID),
|
||||
|
||||
/// Perform a backreference match.
|
||||
BackRef(u32),
|
||||
|
||||
/// Match the next character against the bracket contents, stored at the given index in the CompiledRegex.
|
||||
Bracket(usize),
|
||||
|
||||
/// A simple bitmap bracket for ASCII.
|
||||
/// It contains a bitmap of the range [0, 127].
|
||||
AsciiBracket(AsciiBitmap),
|
||||
|
||||
/// Perform a lookahead assertion.
|
||||
Lookahead {
|
||||
negate: bool,
|
||||
start_group: CaptureGroupID,
|
||||
end_group: CaptureGroupID,
|
||||
continuation: JumpTarget,
|
||||
},
|
||||
|
||||
/// Perform a lookbehind assertion.
|
||||
Lookbehind {
|
||||
negate: bool,
|
||||
start_group: CaptureGroupID,
|
||||
end_group: CaptureGroupID,
|
||||
continuation: JumpTarget,
|
||||
},
|
||||
|
||||
/// \w or \W word boundaries.
|
||||
WordBoundary {
|
||||
invert: bool,
|
||||
},
|
||||
|
||||
/// \w or \W word boundaries in Unicode case-insensitive mode.
|
||||
/// This considers characters that fold to ASCII word chars as word chars,
|
||||
/// per ES9 21.2.2.6.1.
|
||||
WordBoundaryUnicodeICase {
|
||||
invert: bool,
|
||||
},
|
||||
|
||||
/// Match any of the contained chars
|
||||
/// There is no length field; characters are simply duplicated as necessary.
|
||||
CharSet([u32; MAX_CHAR_SET_LENGTH]),
|
||||
|
||||
/// Match the next byte against some possibilities.
|
||||
ByteSet2(ByteArraySet<[u8; 2]>),
|
||||
ByteSet3(ByteArraySet<[u8; 3]>),
|
||||
ByteSet4(ByteArraySet<[u8; 4]>),
|
||||
|
||||
/// Match a sequence of literal bytes.
|
||||
ByteSeq1([u8; 1]),
|
||||
ByteSeq2([u8; 2]),
|
||||
ByteSeq3([u8; 3]),
|
||||
ByteSeq4([u8; 4]),
|
||||
ByteSeq5([u8; 5]),
|
||||
ByteSeq6([u8; 6]),
|
||||
ByteSeq7([u8; 7]),
|
||||
ByteSeq8([u8; 8]),
|
||||
ByteSeq9([u8; 9]),
|
||||
ByteSeq10([u8; 10]),
|
||||
ByteSeq11([u8; 11]),
|
||||
ByteSeq12([u8; 12]),
|
||||
ByteSeq13([u8; 13]),
|
||||
ByteSeq14([u8; 14]),
|
||||
ByteSeq15([u8; 15]),
|
||||
ByteSeq16([u8; 16]),
|
||||
|
||||
/// An instruction that always fails, which may be produced in weird cases
|
||||
/// like an inverted bracket which matches everything.
|
||||
JustFail,
|
||||
}
|
||||
|
||||
/// The peeled prefix start predicate.
|
||||
/// This is a fast way of locating the first potential match.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum StartPredicate {
|
||||
/// May match an arbitrary sequence.
|
||||
Arbitrary,
|
||||
|
||||
/// Look for the first instance of any of the given bytes.
|
||||
ByteSet1([u8; 1]),
|
||||
ByteSet2([u8; 2]),
|
||||
ByteSet3([u8; 3]),
|
||||
|
||||
/// Look for a byte sequence.
|
||||
ByteSeq(Box<memmem::Finder<'static>>),
|
||||
|
||||
/// Look for a byte which matches the bitmap.
|
||||
ByteBracket(ByteBitmap),
|
||||
|
||||
/// The regex is anchored to the start of the line/string.
|
||||
/// This avoids string searching entirely and only tries matching at the beginning.
|
||||
StartAnchored,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CompiledRegex {
|
||||
// Sequence of instructions.
|
||||
pub insns: Vec<Insn>,
|
||||
|
||||
// The bracket contents, indexed by the value of the `Bracket` instruction.
|
||||
pub brackets: Vec<BracketContents>,
|
||||
|
||||
// Predicate to rapidly find the first potential match.
|
||||
pub start_pred: StartPredicate,
|
||||
|
||||
// Number of loops, used to populate loop data.
|
||||
pub loops: u32,
|
||||
|
||||
// Number of capture groups, used to populate capture group data.
|
||||
pub groups: u32,
|
||||
|
||||
// A list of capture group names. This is either:
|
||||
// - Empty, if there were no named capture groups.
|
||||
// - A list of names with length `groups`, corresponding to the capture
|
||||
// group names in order. Groups without names have an empty string.
|
||||
pub group_names: Box<[Box<str>]>,
|
||||
|
||||
// Flags controlling matching.
|
||||
pub flags: api::Flags,
|
||||
}
|
||||
Vendored
+590
@@ -0,0 +1,590 @@
|
||||
//! Intermediate representation for a regex
|
||||
|
||||
use crate::api;
|
||||
use crate::types::{BracketContents, CaptureGroupID, CaptureGroupName};
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{boxed::Box, string::ToString, vec::Vec};
|
||||
use core::fmt;
|
||||
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub enum AnchorType {
|
||||
StartOfLine, // ^
|
||||
EndOfLine, // $
|
||||
}
|
||||
|
||||
/// A Quantifier.
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct Quantifier {
|
||||
/// Minimum number of iterations of the loop, inclusive.
|
||||
pub min: usize,
|
||||
|
||||
/// Maximum number of iterations of the loop, inclusive;
|
||||
/// or None if unbounded.
|
||||
pub max: Option<usize>,
|
||||
|
||||
/// Whether the loop is greedy.
|
||||
pub greedy: bool,
|
||||
}
|
||||
|
||||
/// The node types of our IR.
|
||||
#[derive(Debug)]
|
||||
pub enum Node {
|
||||
/// Matches the empty string.
|
||||
Empty,
|
||||
|
||||
/// Reaching this node terminates the match successfully.
|
||||
Goal,
|
||||
|
||||
/// Match a literal character.
|
||||
/// If icase is true, then `c` MUST be already folded.
|
||||
Char { c: u32, icase: bool },
|
||||
|
||||
/// Match a literal sequence of bytes.
|
||||
ByteSequence(Vec<u8>),
|
||||
|
||||
/// Match any of a set of *bytes*.
|
||||
/// This may not exceed length MAX_BYTE_SET_LENGTH.
|
||||
ByteSet(Vec<u8>),
|
||||
|
||||
/// Match any of a set of *chars*, case-insensitive.
|
||||
/// This may not exceed length MAX_CHAR_SET_LENGTH.
|
||||
CharSet(Vec<u32>),
|
||||
|
||||
/// Match the catenation of multiple nodes.
|
||||
Cat(Vec<Node>),
|
||||
|
||||
/// Match an alternation like a|b.
|
||||
Alt(Box<Node>, Box<Node>),
|
||||
|
||||
/// Match anything including newlines.
|
||||
MatchAny,
|
||||
|
||||
/// Match anything except a newline.
|
||||
MatchAnyExceptLineTerminator,
|
||||
|
||||
/// Match an anchor like ^ or $. The `multiline` flag controls whether the
|
||||
/// anchor should treat line terminators as boundaries.
|
||||
Anchor {
|
||||
anchor_type: AnchorType,
|
||||
multiline: bool,
|
||||
},
|
||||
|
||||
/// Word boundary (\b or \B).
|
||||
WordBoundary { invert: bool },
|
||||
|
||||
/// A capturing group.
|
||||
/// If the name is set, it is guaranteed nonempty.
|
||||
/// Note that multiple capture groups may share the same name in different alternations.
|
||||
CaptureGroup {
|
||||
id: CaptureGroupID,
|
||||
contents: Box<Node>,
|
||||
name: Option<CaptureGroupName>,
|
||||
},
|
||||
|
||||
/// A backreference.
|
||||
/// Indices are 1-based. That is, it matches JS syntax: \1 is the first capture group,
|
||||
/// not everything.
|
||||
/// A backreference that logically may match multiple capture groups (through shared names)
|
||||
/// are emitted through alternations of backreferences.
|
||||
BackRef(u32),
|
||||
|
||||
/// A bracket.
|
||||
Bracket(BracketContents),
|
||||
|
||||
/// A lookaround assertions like (?:) or (?!).
|
||||
LookaroundAssertion {
|
||||
negate: bool,
|
||||
backwards: bool,
|
||||
start_group: CaptureGroupID,
|
||||
end_group: CaptureGroupID,
|
||||
contents: Box<Node>,
|
||||
},
|
||||
|
||||
/// A loop like /.*/ or /x{3, 5}?/
|
||||
Loop {
|
||||
loopee: Box<Node>,
|
||||
quant: Quantifier,
|
||||
enclosed_groups: core::ops::Range<u16>,
|
||||
},
|
||||
|
||||
/// A loop whose body matches exactly one character.
|
||||
/// Enclosed capture groups are forbidden here.
|
||||
Loop1CharBody {
|
||||
loopee: Box<Node>,
|
||||
quant: Quantifier,
|
||||
},
|
||||
}
|
||||
|
||||
pub type NodeList = Vec<Node>;
|
||||
|
||||
impl Node {
|
||||
/// Helper to return an "always fails" node.
|
||||
pub fn make_always_fails() -> Node {
|
||||
Node::CharSet(Vec::new())
|
||||
}
|
||||
|
||||
/// Reverse the children of \p self if in a lookbehind.
|
||||
/// Used as a parameter to walk_mut.
|
||||
pub fn reverse_cats(&mut self, w: &mut Walk) {
|
||||
match self {
|
||||
Node::Cat(nodes) if w.in_lookbehind => nodes.reverse(),
|
||||
Node::ByteSequence(..) => panic!("Should not be reversing literal bytes"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// \return whether this is an Empty node.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
matches!(self, Node::Empty)
|
||||
}
|
||||
|
||||
/// \return whether this is a Cat node.
|
||||
pub fn is_cat(&self) -> bool {
|
||||
matches!(self, Node::Cat(..))
|
||||
}
|
||||
|
||||
/// \return whether this node is known to match exactly one char.
|
||||
/// This is best-effort: a false return is always safe.
|
||||
pub fn matches_exactly_one_char(&self) -> bool {
|
||||
match self {
|
||||
Node::Char { .. } => true,
|
||||
Node::CharSet(contents) => !contents.is_empty(),
|
||||
Node::Bracket(contents) => !contents.is_empty(),
|
||||
Node::MatchAny => true,
|
||||
Node::MatchAnyExceptLineTerminator => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// \return true if this node will always fail to match.
|
||||
/// Note this is different than matching the empty string.
|
||||
/// For example, an empty bracket /[]/ tries to match one char
|
||||
/// from an empty set.
|
||||
pub fn match_always_fails(&self) -> bool {
|
||||
match self {
|
||||
Node::ByteSet(bytes) => bytes.is_empty(),
|
||||
Node::CharSet(contents) => contents.is_empty(),
|
||||
Node::Bracket(contents) => contents.is_empty(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Duplicate a node, perhaps assigning new loop IDs. Note we must never
|
||||
/// copy a capture group.
|
||||
///
|
||||
/// Returns None if the depth is too high.
|
||||
pub fn try_duplicate(&self, mut depth: usize) -> Option<Node> {
|
||||
if depth > 100 {
|
||||
return None;
|
||||
}
|
||||
depth += 1;
|
||||
Some(match self {
|
||||
Node::Empty => Node::Empty,
|
||||
Node::Goal => Node::Goal,
|
||||
&Node::Char { c, icase } => Node::Char { c, icase },
|
||||
Node::ByteSequence(bytes) => Node::ByteSequence(bytes.clone()),
|
||||
Node::ByteSet(bytes) => Node::ByteSet(bytes.clone()),
|
||||
Node::CharSet(chars) => Node::CharSet(chars.clone()),
|
||||
Node::Cat(nodes) => {
|
||||
let mut new_nodes = Vec::with_capacity(nodes.len());
|
||||
for n in nodes {
|
||||
new_nodes.push(n.try_duplicate(depth)?);
|
||||
}
|
||||
Node::Cat(new_nodes)
|
||||
}
|
||||
Node::Alt(left, right) => Node::Alt(
|
||||
Box::new(left.try_duplicate(depth)?),
|
||||
Box::new(right.try_duplicate(depth)?),
|
||||
),
|
||||
Node::MatchAny => Node::MatchAny,
|
||||
Node::MatchAnyExceptLineTerminator => Node::MatchAnyExceptLineTerminator,
|
||||
&Node::Anchor {
|
||||
anchor_type,
|
||||
multiline,
|
||||
} => Node::Anchor {
|
||||
anchor_type,
|
||||
multiline,
|
||||
},
|
||||
|
||||
Node::Loop {
|
||||
loopee,
|
||||
quant,
|
||||
enclosed_groups,
|
||||
} => {
|
||||
assert!(
|
||||
enclosed_groups.start >= enclosed_groups.end,
|
||||
"Cannot duplicate a loop with enclosed groups"
|
||||
);
|
||||
Node::Loop {
|
||||
loopee: Box::new(loopee.as_ref().try_duplicate(depth)?),
|
||||
quant: *quant,
|
||||
enclosed_groups: enclosed_groups.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
Node::Loop1CharBody { loopee, quant } => Node::Loop1CharBody {
|
||||
loopee: Box::new(loopee.as_ref().try_duplicate(depth)?),
|
||||
quant: *quant,
|
||||
},
|
||||
|
||||
Node::CaptureGroup { .. } => {
|
||||
panic!("Refusing to duplicate a capture group");
|
||||
}
|
||||
&Node::WordBoundary { invert } => Node::WordBoundary { invert },
|
||||
&Node::BackRef(idx) => Node::BackRef(idx),
|
||||
Node::Bracket(bc) => Node::Bracket(bc.clone()),
|
||||
// Do not reverse into lookarounds, they already have the right sense.
|
||||
Node::LookaroundAssertion {
|
||||
negate,
|
||||
backwards,
|
||||
start_group,
|
||||
end_group,
|
||||
contents,
|
||||
} => {
|
||||
assert!(
|
||||
start_group >= end_group,
|
||||
"Cannot duplicate an assertion with enclosed groups"
|
||||
);
|
||||
Node::LookaroundAssertion {
|
||||
negate: *negate,
|
||||
backwards: *backwards,
|
||||
start_group: *start_group,
|
||||
end_group: *end_group,
|
||||
contents: Box::new((*contents).try_duplicate(depth)?),
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A helper type for walking.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Walk {
|
||||
// It set to true, skip the children of this node.
|
||||
pub skip_children: bool,
|
||||
|
||||
// The current depth of the walk.
|
||||
pub depth: usize,
|
||||
|
||||
// If true, we are in a lookbehind (and so the cursor will move backwards).
|
||||
pub in_lookbehind: bool,
|
||||
|
||||
// If the regex is in unicode mode.
|
||||
pub unicode: bool,
|
||||
}
|
||||
|
||||
impl Walk {
|
||||
fn new(unicode: bool) -> Self {
|
||||
Self {
|
||||
skip_children: false,
|
||||
depth: 0,
|
||||
in_lookbehind: false,
|
||||
unicode,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Walker<'a, F>
|
||||
where
|
||||
F: FnMut(&Node, &mut Walk),
|
||||
{
|
||||
func: &'a mut F,
|
||||
postorder: bool,
|
||||
walk: Walk,
|
||||
}
|
||||
|
||||
impl<F> Walker<'_, F>
|
||||
where
|
||||
F: FnMut(&Node, &mut Walk),
|
||||
{
|
||||
fn process_children(&mut self, n: &Node) {
|
||||
match n {
|
||||
Node::Empty
|
||||
| Node::Goal
|
||||
| Node::Char { .. }
|
||||
| Node::ByteSequence(..)
|
||||
| Node::ByteSet(..)
|
||||
| Node::CharSet(..)
|
||||
| Node::WordBoundary { .. }
|
||||
| Node::BackRef { .. }
|
||||
| Node::Bracket { .. }
|
||||
| Node::MatchAny
|
||||
| Node::MatchAnyExceptLineTerminator
|
||||
| Node::Anchor { .. } => {}
|
||||
Node::Cat(nodes) => {
|
||||
for node in nodes {
|
||||
self.process(node);
|
||||
}
|
||||
}
|
||||
Node::Alt(left, right) => {
|
||||
self.process(left.as_ref());
|
||||
self.process(right.as_ref());
|
||||
}
|
||||
|
||||
Node::Loop { loopee, .. } | Node::Loop1CharBody { loopee, .. } => self.process(loopee),
|
||||
Node::CaptureGroup { contents, .. } => self.process(contents.as_ref()),
|
||||
|
||||
Node::LookaroundAssertion {
|
||||
backwards,
|
||||
contents,
|
||||
..
|
||||
} => {
|
||||
let saved = self.walk.in_lookbehind;
|
||||
self.walk.in_lookbehind = *backwards;
|
||||
self.process(contents.as_ref());
|
||||
self.walk.in_lookbehind = saved;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn process(&mut self, n: &Node) {
|
||||
self.walk.skip_children = false;
|
||||
if !self.postorder {
|
||||
(self.func)(n, &mut self.walk);
|
||||
}
|
||||
if !self.walk.skip_children {
|
||||
self.walk.depth += 1;
|
||||
self.process_children(n);
|
||||
self.walk.depth -= 1;
|
||||
}
|
||||
if self.postorder {
|
||||
(self.func)(n, &mut self.walk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct MutWalker<'a, F>
|
||||
where
|
||||
F: FnMut(&mut Node, &mut Walk),
|
||||
{
|
||||
func: &'a mut F,
|
||||
postorder: bool,
|
||||
walk: Walk,
|
||||
}
|
||||
|
||||
impl<F> MutWalker<'_, F>
|
||||
where
|
||||
F: FnMut(&mut Node, &mut Walk),
|
||||
{
|
||||
fn process_children(&mut self, n: &mut Node) {
|
||||
match n {
|
||||
Node::Empty
|
||||
| Node::Goal
|
||||
| Node::Char { .. }
|
||||
| Node::ByteSequence(..)
|
||||
| Node::ByteSet(..)
|
||||
| Node::CharSet(..)
|
||||
| Node::MatchAny
|
||||
| Node::MatchAnyExceptLineTerminator
|
||||
| Node::Anchor { .. }
|
||||
| Node::WordBoundary { .. }
|
||||
| Node::BackRef { .. }
|
||||
| Node::Bracket { .. } => {}
|
||||
Node::Cat(nodes) => {
|
||||
nodes.iter_mut().for_each(|node| self.process(node));
|
||||
}
|
||||
Node::Alt(left, right) => {
|
||||
self.process(left.as_mut());
|
||||
self.process(right.as_mut());
|
||||
}
|
||||
|
||||
Node::Loop { loopee, .. } | Node::Loop1CharBody { loopee, .. } => {
|
||||
self.process(loopee);
|
||||
}
|
||||
Node::CaptureGroup { contents, .. } => self.process(contents.as_mut()),
|
||||
|
||||
Node::LookaroundAssertion {
|
||||
backwards,
|
||||
contents,
|
||||
..
|
||||
} => {
|
||||
let saved = self.walk.in_lookbehind;
|
||||
self.walk.in_lookbehind = *backwards;
|
||||
self.process(contents.as_mut());
|
||||
self.walk.in_lookbehind = saved;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn process(&mut self, n: &mut Node) {
|
||||
self.walk.skip_children = false;
|
||||
if !self.postorder {
|
||||
(self.func)(n, &mut self.walk);
|
||||
}
|
||||
if !self.walk.skip_children {
|
||||
self.walk.depth += 1;
|
||||
self.process_children(n);
|
||||
self.walk.depth -= 1;
|
||||
}
|
||||
if self.postorder {
|
||||
(self.func)(n, &mut self.walk);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Call a function on every Node.
|
||||
/// If \p postorder is true, then process children before the node;
|
||||
/// otherwise process children after the node.
|
||||
pub fn walk<F>(postorder: bool, unicode: bool, n: &Node, func: &mut F)
|
||||
where
|
||||
F: FnMut(&Node, &mut Walk),
|
||||
{
|
||||
let mut walker = Walker {
|
||||
func,
|
||||
postorder,
|
||||
walk: Walk::new(unicode),
|
||||
};
|
||||
walker.process(n);
|
||||
}
|
||||
|
||||
/// Call a function on every Node, which may mutate the node.
|
||||
/// If \p postorder is true, then process children before the node;
|
||||
/// otherwise process children after the node.
|
||||
/// If postorder is false, the function should return true to process children,
|
||||
/// false to avoid descending into children. If postorder is true, the return
|
||||
/// value is ignored.
|
||||
pub fn walk_mut<F>(postorder: bool, unicode: bool, n: &mut Node, func: &mut F)
|
||||
where
|
||||
F: FnMut(&mut Node, &mut Walk),
|
||||
{
|
||||
let mut walker = MutWalker {
|
||||
func,
|
||||
postorder,
|
||||
walk: Walk::new(unicode),
|
||||
};
|
||||
walker.process(n);
|
||||
}
|
||||
|
||||
/// A regex in IR form.
|
||||
pub struct Regex {
|
||||
pub node: Node,
|
||||
pub flags: api::Flags,
|
||||
}
|
||||
|
||||
impl Regex {}
|
||||
|
||||
fn display_node(node: &Node, depth: usize, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
for _ in 0..depth {
|
||||
write!(f, "..")?;
|
||||
}
|
||||
match node {
|
||||
Node::Empty => {
|
||||
writeln!(f, "Empty")?;
|
||||
}
|
||||
Node::Goal => {
|
||||
writeln!(f, "Goal")?;
|
||||
}
|
||||
Node::Char { c, icase: _ } => {
|
||||
writeln!(f, "'{}'", &c.to_string())?;
|
||||
}
|
||||
Node::ByteSequence(bytes) => {
|
||||
write!(f, "ByteSeq{} 0x", bytes.len())?;
|
||||
for &b in bytes {
|
||||
write!(f, "{:x}", b)?;
|
||||
}
|
||||
writeln!(f)?;
|
||||
}
|
||||
Node::ByteSet(bytes) => {
|
||||
let len = bytes.len();
|
||||
write!(f, "ByteSet{}", len)?;
|
||||
for &b in bytes {
|
||||
write!(f, " 0x{:x}", b)?;
|
||||
}
|
||||
writeln!(f)?;
|
||||
}
|
||||
Node::CharSet(chars) => {
|
||||
write!(f, "CharSet ")?;
|
||||
let mut first = true;
|
||||
for &c in chars {
|
||||
if !first {
|
||||
write!(f, ", ")?;
|
||||
}
|
||||
first = false;
|
||||
write!(f, "0x{:x}", { c })?;
|
||||
}
|
||||
writeln!(f)?;
|
||||
}
|
||||
Node::Cat(..) => {
|
||||
writeln!(f, "Cat")?;
|
||||
}
|
||||
Node::Alt(..) => {
|
||||
writeln!(f, "Alt")?;
|
||||
}
|
||||
Node::MatchAny => {
|
||||
writeln!(f, "MatchAny")?;
|
||||
}
|
||||
Node::MatchAnyExceptLineTerminator => {
|
||||
writeln!(f, "MatchAnyExceptLineTerminator")?;
|
||||
}
|
||||
Node::Anchor {
|
||||
anchor_type,
|
||||
multiline,
|
||||
} => {
|
||||
writeln!(f, "Anchor {:?} multiline={}", anchor_type, multiline)?;
|
||||
}
|
||||
Node::Loop {
|
||||
quant,
|
||||
enclosed_groups,
|
||||
..
|
||||
} => {
|
||||
writeln!(f, "Loop (groups {:?}) {:?}", enclosed_groups, quant)?;
|
||||
}
|
||||
Node::Loop1CharBody { quant, .. } => {
|
||||
writeln!(f, "Loop1Char {:?}", quant)?;
|
||||
}
|
||||
Node::CaptureGroup { id, name, .. } => {
|
||||
if let Some(name) = name {
|
||||
writeln!(f, "CaptureGroup {:?} {:?}", id, name)?;
|
||||
} else {
|
||||
writeln!(f, "CaptureGroup {:?}", id)?;
|
||||
}
|
||||
}
|
||||
&Node::WordBoundary { invert } => {
|
||||
let kind = if invert { "\\B" } else { "\\b" };
|
||||
writeln!(f, "WordBoundary {:?} ", kind)?;
|
||||
}
|
||||
&Node::BackRef(group) => {
|
||||
writeln!(f, "BackRef {:?} ", group)?;
|
||||
}
|
||||
Node::Bracket(contents) => {
|
||||
writeln!(f, "Bracket {:?}", contents)?;
|
||||
}
|
||||
|
||||
&Node::LookaroundAssertion {
|
||||
negate,
|
||||
backwards,
|
||||
start_group,
|
||||
end_group,
|
||||
..
|
||||
} => {
|
||||
let sense = if negate { "negative" } else { "positive" };
|
||||
let direction = if backwards { "backwards" } else { "forwards" };
|
||||
writeln!(
|
||||
f,
|
||||
"LookaroundAssertion {} {} {:?} {:?}",
|
||||
sense, direction, start_group, end_group
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl fmt::Display for Regex {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
//display_node(&self.node, 0, f)
|
||||
let mut result = Ok(());
|
||||
walk(
|
||||
false,
|
||||
self.flags.unicode,
|
||||
&self.node,
|
||||
&mut |node: &Node, walk: &mut Walk| {
|
||||
if result.is_ok() {
|
||||
result = display_node(node, walk.depth, f)
|
||||
}
|
||||
},
|
||||
);
|
||||
result
|
||||
}
|
||||
}
|
||||
Vendored
+187
@@ -0,0 +1,187 @@
|
||||
/*!
|
||||
|
||||
# regress - REGex in Rust with EcmaScript Syntax
|
||||
|
||||
This crate provides a regular expression engine which targets EcmaScript (aka JavaScript) regular expression syntax.
|
||||
|
||||
# Example: test if a string contains a match
|
||||
|
||||
```rust
|
||||
use regress::Regex;
|
||||
let re = Regex::new(r"\d{4}").unwrap();
|
||||
let matched = re.find("2020-20-05").is_some();
|
||||
assert!(matched);
|
||||
```
|
||||
|
||||
# Example: iterating over matches
|
||||
|
||||
Here we use a backreference to find doubled characters:
|
||||
|
||||
```rust
|
||||
use regress::Regex;
|
||||
let re = Regex::new(r"(\w)\1").unwrap();
|
||||
let text = "Frankly, Miss Piggy, I don't give a hoot!";
|
||||
for m in re.find_iter(text) {
|
||||
println!("{}", &text[m.range()])
|
||||
}
|
||||
// Output: ss
|
||||
// Output: gg
|
||||
// Output: oo
|
||||
|
||||
```
|
||||
|
||||
# Example: using capture groups
|
||||
|
||||
Capture groups are available in the `Match` object produced by a successful match.
|
||||
A capture group is a range of byte indexes into the original string.
|
||||
|
||||
```rust
|
||||
use regress::Regex;
|
||||
let re = Regex::new(r"(\d{4})").unwrap();
|
||||
let text = "Today is 2020-20-05";
|
||||
let m = re.find(text).unwrap();
|
||||
let group = m.group(1).unwrap();
|
||||
println!("Year: {}", &text[group]);
|
||||
// Output: Year: 2020
|
||||
```
|
||||
|
||||
# Example: using with Pattern trait (nightly only)
|
||||
|
||||
When the `pattern` feature is enabled and using nightly Rust, `Regex` can be used with standard string methods:
|
||||
|
||||
```rust,ignore
|
||||
#![feature(pattern)]
|
||||
use regress::Regex;
|
||||
let re = Regex::new(r"\d+").unwrap();
|
||||
let text = "abc123def456";
|
||||
|
||||
// Use with str methods
|
||||
assert_eq!(text.find(&re), Some(3));
|
||||
assert!(text.contains(&re));
|
||||
let parts: Vec<&str> = text.split(&re).collect();
|
||||
assert_eq!(parts, vec!["abc", "def", ""]);
|
||||
```
|
||||
|
||||
# Example: escaping strings for literal matching
|
||||
|
||||
Use the `escape` function to escape special regex characters in a string:
|
||||
|
||||
```rust
|
||||
use regress::{escape, Regex};
|
||||
let user_input = "How much $ do you have? (in dollars)";
|
||||
let escaped = escape(user_input);
|
||||
let re = Regex::new(&escaped).unwrap();
|
||||
assert!(re.find(user_input).is_some());
|
||||
```
|
||||
|
||||
# Supported Syntax
|
||||
|
||||
regress targets ES 2018 syntax. You can refer to the many resources about JavaScript regex syntax.
|
||||
|
||||
There are some features which have yet to be implemented:
|
||||
|
||||
- Named character classes liks `[[:alpha:]]`
|
||||
- Unicode property escapes like `\p{Sc}`
|
||||
|
||||
Note the parser assumes the `u` (Unicode) flag, as the non-Unicode path is tied to JS's UCS-2 string encoding and the semantics cannot be usefully expressed in Rust.
|
||||
|
||||
# Unicode remarks
|
||||
|
||||
regress supports Unicode case folding. For example:
|
||||
|
||||
```rust
|
||||
use regress::Regex;
|
||||
let re = Regex::with_flags("\u{00B5}", "i").unwrap();
|
||||
assert!(re.find("\u{03BC}").is_some());
|
||||
```
|
||||
|
||||
Here the U+00B5 (micro sign) was case-insensitively matched against U+03BC (small letter mu).
|
||||
|
||||
regress does NOT perform normalization. For example, e-with-accute-accent can be precomposed or decomposed, and these are treated as not equivalent:
|
||||
|
||||
```rust
|
||||
use regress::{Regex, Flags};
|
||||
let re = Regex::new("\u{00E9}").unwrap();
|
||||
assert!(re.find("\u{0065}\u{0301}").is_none());
|
||||
```
|
||||
|
||||
This agrees with JavaScript semantics. Perform any required normalization before regex matching.
|
||||
|
||||
## Ascii matching
|
||||
|
||||
regress has an "ASCII mode" which treats each 8-bit quantity as a separate character.
|
||||
This may provide improved performance if you do not need Unicode semantics, because it can avoid decoding UTF-8 and has simpler (ASCII-only) case-folding.
|
||||
|
||||
Example:
|
||||
|
||||
```rust
|
||||
use regress::Regex;
|
||||
let re = Regex::with_flags("BC", "i").unwrap();
|
||||
assert!(re.find("abcd").is_some());
|
||||
```
|
||||
|
||||
# Comparison to regex crate
|
||||
|
||||
regress supports features (required by the EcmaScript spec) that regex does not, including backreferences and zero-width lookaround assertions.
|
||||
However the regex crate provides linear-time matching guarantees, while regress does not. This difference is due
|
||||
to the architecture: regex uses finite automata while regress uses "classical backtracking."
|
||||
|
||||
|
||||
# Architecture
|
||||
|
||||
regress has a parser, intermediate representation, optimizer which acts on the IR, bytecode emitter, and two bytecode interpreters, referred to as "backends".
|
||||
|
||||
The major interpreter is the "classical backtracking" which uses an explicit backtracking stack, similar to JS implementations. There is also the "PikeVM" pseudo-toy backend which is mainly used for testing and verification.
|
||||
|
||||
# Crate features
|
||||
|
||||
- **utf16**. When enabled, additional APIs are made available that allow matching text formatted in UTF-16 and UCS-2 (`&[u16]`) without going through a conversion to and from UTF-8 (`&str`) first. This is particularly useful when interacting with and/or (re)implementing existing systems that use those encodings, such as JavaScript, Windows, and the JVM.
|
||||
|
||||
- **pattern**. When enabled (nightly only), implements the `std::str::pattern::Pattern` trait for `Regex`, allowing it to be used with standard string methods like `str::find`, `str::contains`, `str::split`, etc.
|
||||
|
||||
*/
|
||||
|
||||
#![cfg_attr(not(feature = "std"), no_std)]
|
||||
#![cfg_attr(feature = "pattern", feature(pattern))]
|
||||
#![warn(clippy::all)]
|
||||
#![allow(
|
||||
clippy::upper_case_acronyms,
|
||||
clippy::match_like_matches_macro,
|
||||
clippy::uninlined_format_args,
|
||||
clippy::collapsible_if
|
||||
)]
|
||||
// Clippy's manual_range_contains suggestion produces worse codegen.
|
||||
#![allow(clippy::manual_range_contains)]
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
#[macro_use]
|
||||
extern crate alloc;
|
||||
|
||||
pub use crate::api::*;
|
||||
|
||||
#[macro_use]
|
||||
mod util;
|
||||
|
||||
mod api;
|
||||
mod bytesearch;
|
||||
mod charclasses;
|
||||
mod classicalbacktrack;
|
||||
mod codepointset;
|
||||
mod cursor;
|
||||
mod emit;
|
||||
mod exec;
|
||||
mod indexing;
|
||||
mod insn;
|
||||
mod ir;
|
||||
mod matchers;
|
||||
mod optimizer;
|
||||
mod parse;
|
||||
mod position;
|
||||
mod scm;
|
||||
mod startpredicate;
|
||||
mod types;
|
||||
mod unicode;
|
||||
mod unicodetables;
|
||||
|
||||
#[cfg(feature = "backend-pikevm")]
|
||||
mod pikevm;
|
||||
Vendored
+113
@@ -0,0 +1,113 @@
|
||||
use crate::cursor::Direction;
|
||||
use crate::indexing::{ElementType, InputIndexer};
|
||||
use crate::types::BracketContents;
|
||||
use crate::unicodetables::nonascii_folds_to_ascii_word_char;
|
||||
use crate::{cursor, unicode};
|
||||
|
||||
pub trait CharProperties {
|
||||
type Element: ElementType;
|
||||
|
||||
/// Case-fold an element.
|
||||
fn fold(c: Self::Element, unicode: bool) -> Self::Element;
|
||||
|
||||
/// \return whether this is a word char.
|
||||
/// ES9 21.2.2.6.2.
|
||||
fn is_word_char(c: Self::Element) -> bool {
|
||||
let c = c.as_u32();
|
||||
'a' as u32 <= c && c <= 'z' as u32
|
||||
|| 'A' as u32 <= c && c <= 'Z' as u32
|
||||
|| '0' as u32 <= c && c <= '9' as u32
|
||||
|| c == '_' as u32
|
||||
}
|
||||
|
||||
/// \return whether this is a word char in Unicode case-insensitive mode.
|
||||
/// This includes ASCII word chars and any character that folds to an ASCII word char.
|
||||
fn is_word_char_unicode_icase(c: Self::Element) -> bool {
|
||||
Self::is_word_char(c) || nonascii_folds_to_ascii_word_char(c.as_u32())
|
||||
}
|
||||
|
||||
/// ES9 11.3
|
||||
fn is_line_terminator(c: Self::Element) -> bool {
|
||||
matches!(c.as_u32(), 0x000A | 0x000D | 0x2028 | 0x2029)
|
||||
}
|
||||
|
||||
/// \return whether the bracket \p bc matches the given character \p c,
|
||||
/// respecting case. Respects 'invert'.
|
||||
#[inline(always)]
|
||||
fn bracket(bc: &BracketContents, cp: Self::Element) -> bool {
|
||||
let cp = cp.into();
|
||||
if bc.cps.contains(cp) {
|
||||
return !bc.invert;
|
||||
}
|
||||
bc.invert
|
||||
}
|
||||
}
|
||||
|
||||
pub struct UTF8CharProperties {}
|
||||
|
||||
impl CharProperties for UTF8CharProperties {
|
||||
type Element = char;
|
||||
|
||||
fn fold(c: Self::Element, unicode: bool) -> Self::Element {
|
||||
char::from_u32(unicode::fold_code_point(c.into(), unicode)).unwrap_or(c)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ASCIICharProperties {}
|
||||
impl CharProperties for ASCIICharProperties {
|
||||
type Element = u8;
|
||||
|
||||
fn fold(c: Self::Element, unicode: bool) -> Self::Element {
|
||||
if unicode {
|
||||
c.to_ascii_lowercase()
|
||||
} else {
|
||||
c.to_ascii_uppercase()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "utf16")]
|
||||
pub struct Utf16CharProperties {}
|
||||
|
||||
#[cfg(feature = "utf16")]
|
||||
impl CharProperties for Utf16CharProperties {
|
||||
type Element = u32;
|
||||
|
||||
fn fold(c: Self::Element, unicode: bool) -> Self::Element {
|
||||
unicode::fold_code_point(c, unicode)
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether the \p orig_range within \p cursor matches position \p pos.
|
||||
pub fn backref<Input: InputIndexer, Dir: Direction>(
|
||||
input: &Input,
|
||||
dir: Dir,
|
||||
orig_range: core::ops::Range<Input::Position>,
|
||||
pos: &mut Input::Position,
|
||||
) -> bool {
|
||||
input.subrange_eq(dir, pos, orig_range)
|
||||
}
|
||||
|
||||
pub fn backref_icase<Input: InputIndexer, Dir: Direction>(
|
||||
input: &Input,
|
||||
dir: Dir,
|
||||
orig_range: core::ops::Range<Input::Position>,
|
||||
pos: &mut Input::Position,
|
||||
) -> bool {
|
||||
let ref_input = input.subinput(orig_range);
|
||||
let mut ref_pos = if Dir::FORWARD {
|
||||
ref_input.left_end()
|
||||
} else {
|
||||
ref_input.right_end()
|
||||
};
|
||||
while let Some(c1) = cursor::next(&ref_input, dir, &mut ref_pos) {
|
||||
let mut matched = false;
|
||||
if let Some(c2) = cursor::next(input, dir, pos) {
|
||||
matched = input.fold_equals(c1, c2)
|
||||
}
|
||||
if !matched {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
Vendored
+544
@@ -0,0 +1,544 @@
|
||||
//! Optimizations on regex IR
|
||||
|
||||
use crate::insn::{MAX_BYTE_SET_LENGTH, MAX_CHAR_SET_LENGTH};
|
||||
use crate::ir::*;
|
||||
use crate::types::BracketContents;
|
||||
use crate::unicode;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{boxed::Box, vec::Vec};
|
||||
|
||||
/// When unrolling a loop, the largest minimum count we will unroll.
|
||||
const LOOP_UNROLL_THRESHOLD: usize = 5;
|
||||
|
||||
/// Things that a Pass may do.
|
||||
pub enum PassAction {
|
||||
// Do nothing to the given node.
|
||||
Keep,
|
||||
|
||||
// Notes that we modified the node in-place.
|
||||
Modified,
|
||||
|
||||
// Remove the given node outright, effectively replacing it with empty.
|
||||
Remove,
|
||||
|
||||
/// Replace the given node with a new Node.
|
||||
Replace(Node),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Pass<'a, F>
|
||||
where
|
||||
F: FnMut(&mut Node, &Walk) -> PassAction,
|
||||
{
|
||||
// The function.
|
||||
func: &'a mut F,
|
||||
|
||||
// Whether this pass has changed anything.
|
||||
changed: bool,
|
||||
|
||||
// If the regex is in unicode mode.
|
||||
unicode: bool,
|
||||
}
|
||||
|
||||
impl<'a, F> Pass<'a, F>
|
||||
where
|
||||
F: FnMut(&mut Node, &Walk) -> PassAction,
|
||||
{
|
||||
fn new(func: &'a mut F, unicode: bool) -> Self {
|
||||
Pass {
|
||||
func,
|
||||
changed: false,
|
||||
unicode,
|
||||
}
|
||||
}
|
||||
|
||||
fn run_postorder(&mut self, start: &mut Node) {
|
||||
walk_mut(
|
||||
true,
|
||||
self.unicode,
|
||||
start,
|
||||
&mut |n: &mut Node, walk: &mut Walk| match (self.func)(n, walk) {
|
||||
PassAction::Keep => {}
|
||||
PassAction::Modified => {
|
||||
self.changed = true;
|
||||
}
|
||||
PassAction::Remove => {
|
||||
*n = Node::Empty;
|
||||
self.changed = true;
|
||||
}
|
||||
PassAction::Replace(newnode) => {
|
||||
*n = newnode;
|
||||
self.changed = true;
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn run_to_fixpoint(&mut self, n: &mut Node) {
|
||||
debug_assert!(!self.changed, "Pass has already been run");
|
||||
loop {
|
||||
self.changed = false;
|
||||
self.run_postorder(n);
|
||||
if !self.changed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run a "pass" on a regex, which is a function that takes a Node and maybe
|
||||
/// returns a new node. \return true if something changed, false if nothing did.
|
||||
fn run_pass<F>(r: &mut Regex, func: &mut F) -> bool
|
||||
where
|
||||
F: FnMut(&mut Node, &Walk) -> PassAction,
|
||||
{
|
||||
let mut p = Pass::new(func, r.flags.unicode);
|
||||
p.run_to_fixpoint(&mut r.node);
|
||||
p.changed
|
||||
}
|
||||
|
||||
// Here are some optimizations we support.
|
||||
|
||||
// Remove empty Nodes.
|
||||
fn remove_empties(n: &mut Node, _w: &Walk) -> PassAction {
|
||||
match n {
|
||||
Node::Empty | Node::Goal | Node::Char { .. } => PassAction::Keep,
|
||||
Node::ByteSequence(v) => {
|
||||
if v.is_empty() {
|
||||
PassAction::Remove
|
||||
} else {
|
||||
PassAction::Keep
|
||||
}
|
||||
}
|
||||
|
||||
// Note: do not remove empty sets. These always match against one character; an empty set
|
||||
// should just fail.
|
||||
Node::ByteSet(..) | Node::CharSet(..) => PassAction::Keep,
|
||||
Node::Cat(nodes) => {
|
||||
let blen = nodes.len();
|
||||
nodes.retain(|nn| !nn.is_empty());
|
||||
if nodes.len() == blen {
|
||||
// Nothing was removed.
|
||||
PassAction::Keep
|
||||
} else {
|
||||
match nodes.len() {
|
||||
0 => PassAction::Remove,
|
||||
1 => PassAction::Replace(nodes.pop().unwrap()),
|
||||
_ => PassAction::Modified,
|
||||
}
|
||||
}
|
||||
}
|
||||
Node::Alt(left, right) => {
|
||||
// Empty alt may match the empty string.
|
||||
// Remove it only if both sides are empty.
|
||||
if left.is_empty() && right.is_empty() {
|
||||
PassAction::Remove
|
||||
} else {
|
||||
PassAction::Keep
|
||||
}
|
||||
}
|
||||
Node::MatchAny | Node::MatchAnyExceptLineTerminator | Node::Anchor { .. } => {
|
||||
PassAction::Keep
|
||||
}
|
||||
Node::Loop {
|
||||
quant,
|
||||
loopee,
|
||||
enclosed_groups,
|
||||
} => {
|
||||
// A loop is empty if it has an empty body, or 0 max iters.
|
||||
// But do not remove contained capture groups.
|
||||
if loopee.is_empty()
|
||||
|| (quant.max == Some(0) && enclosed_groups.start == enclosed_groups.end)
|
||||
{
|
||||
PassAction::Remove
|
||||
} else {
|
||||
PassAction::Keep
|
||||
}
|
||||
}
|
||||
Node::Loop1CharBody { .. } => PassAction::Keep,
|
||||
Node::CaptureGroup { .. } => {
|
||||
// Capture groups could in principle be optimized if they only match empties.
|
||||
PassAction::Keep
|
||||
}
|
||||
Node::WordBoundary { .. } | Node::BackRef { .. } | Node::Bracket { .. } => PassAction::Keep,
|
||||
Node::LookaroundAssertion {
|
||||
negate, contents, ..
|
||||
} => {
|
||||
// Negative arounds that match empties could in principle be optimized to always
|
||||
// fail. Here we only optimize positive ones.
|
||||
if !*negate && contents.is_empty() {
|
||||
PassAction::Remove
|
||||
} else {
|
||||
PassAction::Keep
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a node contains any capture groups (direct or nested)
|
||||
fn contains_capture_groups(node: &Node) -> bool {
|
||||
match node {
|
||||
Node::CaptureGroup { .. } => true,
|
||||
Node::Cat(nodes) => nodes.iter().any(contains_capture_groups),
|
||||
Node::Alt(left, right) => contains_capture_groups(left) || contains_capture_groups(right),
|
||||
Node::Loop { loopee, .. } => contains_capture_groups(loopee),
|
||||
Node::LookaroundAssertion { contents, .. } => contains_capture_groups(contents),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
// If a node can never match, replace it with an always fails node.
|
||||
fn propagate_early_fails(n: &mut Node, _w: &Walk) -> PassAction {
|
||||
// Don't optimize nodes containing capture groups to preserve user-visible group numbers
|
||||
if contains_capture_groups(n) {
|
||||
return PassAction::Keep;
|
||||
}
|
||||
|
||||
match n {
|
||||
Node::Cat(nodes) => {
|
||||
// If any child is an early fail, we are an early fail.
|
||||
// Note this assumes that there is no node after a Goal node.
|
||||
if nodes.iter().any(|nn| nn.match_always_fails()) {
|
||||
PassAction::Replace(Node::make_always_fails())
|
||||
} else {
|
||||
PassAction::Keep
|
||||
}
|
||||
}
|
||||
Node::Alt(left, right) => {
|
||||
// If both sides are early fails, we are an early fail.
|
||||
let left_fails = left.match_always_fails();
|
||||
let right_fails = right.match_always_fails();
|
||||
match (left_fails, right_fails) {
|
||||
(true, true) => PassAction::Replace(Node::make_always_fails()),
|
||||
(false, false) => PassAction::Keep,
|
||||
(true, false) | (false, true) => {
|
||||
// Here either our left or right node always fails.
|
||||
// "Steal" the other and return it, replacing us.
|
||||
let mut new_node = Node::Empty;
|
||||
core::mem::swap(
|
||||
&mut new_node,
|
||||
if left_fails { &mut *right } else { &mut *left },
|
||||
);
|
||||
PassAction::Replace(new_node)
|
||||
}
|
||||
}
|
||||
}
|
||||
Node::Loop {
|
||||
loopee,
|
||||
quant,
|
||||
enclosed_groups,
|
||||
} => {
|
||||
if enclosed_groups.start < enclosed_groups.end {
|
||||
return PassAction::Keep;
|
||||
}
|
||||
// If the loop body always fails, we always fail.
|
||||
if quant.min > 0 && loopee.match_always_fails() {
|
||||
PassAction::Replace(Node::make_always_fails())
|
||||
} else {
|
||||
PassAction::Keep
|
||||
}
|
||||
}
|
||||
_ => PassAction::Keep,
|
||||
}
|
||||
}
|
||||
|
||||
// Remove excess cats.
|
||||
fn decat(n: &mut Node, _w: &Walk) -> PassAction {
|
||||
match n {
|
||||
Node::Cat(nodes) => {
|
||||
if nodes.is_empty() {
|
||||
PassAction::Remove
|
||||
} else if nodes.len() == 1 {
|
||||
PassAction::Replace(nodes.pop().unwrap())
|
||||
} else if nodes.iter().any(|nn| nn.is_cat()) {
|
||||
// Flatmap child cats.
|
||||
// Unfortunately we can't use flatmap() because there's no single iterator type
|
||||
// we can return.
|
||||
// Avoid copying nodes by switching them into owned vec.
|
||||
let mut catted = Vec::new();
|
||||
core::mem::swap(nodes, &mut catted);
|
||||
|
||||
// Decat them.
|
||||
let mut decatted = Vec::new();
|
||||
for nn in catted {
|
||||
match nn {
|
||||
Node::Cat(mut nnodes) => {
|
||||
decatted.append(&mut nnodes);
|
||||
}
|
||||
_ => decatted.push(nn),
|
||||
}
|
||||
}
|
||||
PassAction::Replace(Node::Cat(decatted))
|
||||
} else {
|
||||
PassAction::Keep
|
||||
}
|
||||
}
|
||||
_ => PassAction::Keep,
|
||||
}
|
||||
}
|
||||
|
||||
/// Unfold icase chars.
|
||||
/// That means for case-insensitive characters, figure out everything that they
|
||||
/// could match.
|
||||
/// TODO: evaluate unfolding performance and consider a cache within the optimizer.
|
||||
fn unfold_icase_chars(n: &mut Node, w: &Walk) -> PassAction {
|
||||
match *n {
|
||||
Node::Char { c, icase } if icase && !w.unicode => {
|
||||
let unfolded = unicode::unfold_uppercase_char(c);
|
||||
debug_assert!(
|
||||
unfolded.contains(&c),
|
||||
"Char should always unfold to at least itself"
|
||||
);
|
||||
match unfolded.len() {
|
||||
0 => panic!("Char should always unfold to at least itself"),
|
||||
1 => {
|
||||
// Character does not fold or unfold at all.
|
||||
PassAction::Replace(Node::Char { c, icase: false })
|
||||
}
|
||||
2..=MAX_BYTE_SET_LENGTH => {
|
||||
// We unfolded to 2+ characters.
|
||||
PassAction::Replace(Node::CharSet(unfolded))
|
||||
}
|
||||
_ => panic!("Unfolded to more characters than we believed possible"),
|
||||
}
|
||||
}
|
||||
Node::Char { c, icase } if icase => {
|
||||
let unfolded = unicode::unfold_char(c);
|
||||
debug_assert!(
|
||||
unfolded.contains(&c),
|
||||
"Char should always unfold to at least itself"
|
||||
);
|
||||
match unfolded.len() {
|
||||
0 => panic!("Char should always unfold to at least itself"),
|
||||
1 => {
|
||||
// Character does not fold or unfold at all.
|
||||
PassAction::Replace(Node::Char { c, icase: false })
|
||||
}
|
||||
2..=MAX_BYTE_SET_LENGTH => {
|
||||
// We unfolded to 2+ characters.
|
||||
PassAction::Replace(Node::CharSet(unfolded))
|
||||
}
|
||||
_ => panic!("Unfolded to more characters than we believed possible"),
|
||||
}
|
||||
}
|
||||
_ => PassAction::Keep,
|
||||
}
|
||||
}
|
||||
|
||||
// Perform simple unrolling of loops that have a minimum.
|
||||
fn unroll_loops(n: &mut Node, _w: &Walk) -> PassAction {
|
||||
match n {
|
||||
Node::Loop {
|
||||
loopee,
|
||||
quant,
|
||||
enclosed_groups,
|
||||
} => {
|
||||
// TODO: consider ignoring loops with nested sub-loops?
|
||||
|
||||
// Do not unroll loops with enclosed groups.
|
||||
if enclosed_groups.start < enclosed_groups.end {
|
||||
return PassAction::Keep;
|
||||
}
|
||||
// Do not unroll large loops, or loops which may execute zero times.
|
||||
if quant.min == 0 || quant.min > LOOP_UNROLL_THRESHOLD {
|
||||
return PassAction::Keep;
|
||||
}
|
||||
|
||||
// We made it through. Replace us with a cat.
|
||||
let mut unrolled = Vec::new();
|
||||
for _ in 0..quant.min {
|
||||
let Some(node) = loopee.try_duplicate(0) else {
|
||||
return PassAction::Keep;
|
||||
};
|
||||
unrolled.push(node);
|
||||
}
|
||||
|
||||
// We unrolled 'min' elements.
|
||||
// Maybe our loop is now empty.
|
||||
quant.max = quant.max.map(|v| v - quant.min);
|
||||
quant.min = 0;
|
||||
if quant.max != Some(0) {
|
||||
// Move the loop to the end of unrolled.
|
||||
let mut loop_node = Node::Empty;
|
||||
core::mem::swap(&mut loop_node, n);
|
||||
unrolled.push(loop_node);
|
||||
}
|
||||
*n = Node::Cat(unrolled);
|
||||
PassAction::Modified
|
||||
}
|
||||
_ => PassAction::Keep,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace Loops with 1Char loops whenever possible.
|
||||
fn promote_1char_loops(n: &mut Node, _w: &Walk) -> PassAction {
|
||||
match n {
|
||||
Node::Loop {
|
||||
loopee,
|
||||
quant,
|
||||
enclosed_groups,
|
||||
} => {
|
||||
// Must be 1Char.
|
||||
if !loopee.matches_exactly_one_char() {
|
||||
return PassAction::Keep;
|
||||
}
|
||||
|
||||
// The above check should be sufficient to ensure we have no enclosed groups.
|
||||
assert!(
|
||||
enclosed_groups.start >= enclosed_groups.end,
|
||||
"Should have no enclosed groups"
|
||||
);
|
||||
|
||||
// This feels hackish?
|
||||
let mut new_loopee = Box::new(Node::Empty);
|
||||
core::mem::swap(&mut new_loopee, loopee);
|
||||
|
||||
*n = Node::Loop1CharBody {
|
||||
loopee: new_loopee,
|
||||
quant: *quant,
|
||||
};
|
||||
PassAction::Modified
|
||||
}
|
||||
_ => PassAction::Keep,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace Cat(Char) with ByteSeq.
|
||||
/// Also replace chars with literal bytes.
|
||||
/// Don't do this in utf16 mode because UTF-16 should never match against bytes.
|
||||
/// TODO: this seems to do too much; consider breaking this up.
|
||||
#[cfg(not(feature = "utf16"))]
|
||||
fn form_literal_bytes(n: &mut Node, walk: &Walk) -> PassAction {
|
||||
// Helper to return a mutable reference to the nodes of a literal bytes.
|
||||
fn get_literal_bytes(n: &mut Node) -> Option<&mut Vec<u8>> {
|
||||
match n {
|
||||
Node::ByteSequence(v) => Some(v),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
match n {
|
||||
Node::Char { c, icase } if !*icase => {
|
||||
if let Some(c) = char::from_u32(*c) {
|
||||
let mut buff = [0; 4];
|
||||
PassAction::Replace(Node::ByteSequence(
|
||||
c.encode_utf8(&mut buff).as_bytes().to_vec(),
|
||||
))
|
||||
} else {
|
||||
PassAction::Keep
|
||||
}
|
||||
}
|
||||
Node::CharSet(chars) if chars.iter().all(|&c| c <= 0x7F) => {
|
||||
// All of our chars are ASCII; use a byte set instead.
|
||||
PassAction::Replace(Node::ByteSet(chars.iter().map(|&c| c as u8).collect()))
|
||||
}
|
||||
Node::Cat(nodes) => {
|
||||
// Find and merge adjacent ByteSeq.
|
||||
let mut modified = false;
|
||||
for idx in 1..nodes.len() {
|
||||
let (prev_slice, curr_slice) = nodes.split_at_mut(idx);
|
||||
match (
|
||||
get_literal_bytes(prev_slice.last_mut().unwrap()),
|
||||
get_literal_bytes(curr_slice.first_mut().unwrap()),
|
||||
) {
|
||||
(Some(prev_bytes), Some(curr_bytes))
|
||||
if !prev_bytes.is_empty() && !curr_bytes.is_empty() =>
|
||||
{
|
||||
if walk.in_lookbehind {
|
||||
// Our characters were already reversed; we need to reverse them again
|
||||
// as literal bytes are always forwards. For
|
||||
// example, if we have (<=ab), then we will get Cat(b, a) but we want
|
||||
// literal bytes "ab".
|
||||
curr_bytes.append(prev_bytes);
|
||||
} else {
|
||||
prev_bytes.append(curr_bytes);
|
||||
core::mem::swap(prev_bytes, curr_bytes);
|
||||
}
|
||||
modified = true;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
if modified {
|
||||
PassAction::Modified
|
||||
} else {
|
||||
PassAction::Keep
|
||||
}
|
||||
}
|
||||
_ => PassAction::Keep,
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to reduce a bracket to something simpler.
|
||||
fn try_reduce_bracket(bc: &BracketContents) -> Option<Node> {
|
||||
if bc.invert {
|
||||
// Give up.
|
||||
return None;
|
||||
}
|
||||
|
||||
// Count the number of code points.
|
||||
let mut cps_count = 0;
|
||||
for iv in bc.cps.intervals() {
|
||||
cps_count += iv.count_codepoints();
|
||||
}
|
||||
if cps_count > MAX_CHAR_SET_LENGTH {
|
||||
// Too many code points.
|
||||
return None;
|
||||
}
|
||||
// Ok, we want to make a char set.
|
||||
// Note we cannot make a char out of surrogates; should char conversion fail we
|
||||
// just give up.
|
||||
let mut res = Vec::new();
|
||||
for iv in bc.cps.intervals() {
|
||||
for cp in iv.codepoints() {
|
||||
res.push(cp);
|
||||
}
|
||||
}
|
||||
debug_assert!(res.len() <= MAX_CHAR_SET_LENGTH, "Unexpectedly many chars");
|
||||
Some(Node::CharSet(res))
|
||||
}
|
||||
|
||||
/// Optimize brackets like [a-b].
|
||||
/// Optimize certain stupid brackets like `[a]` to a single char.
|
||||
/// Invert a bracket if it would *reduce* the number of ranges.
|
||||
/// Note we only run this once.
|
||||
fn simplify_brackets(n: &mut Node, _walk: &Walk) -> PassAction {
|
||||
match n {
|
||||
Node::Bracket(bc) => {
|
||||
if let Some(new_node) = try_reduce_bracket(bc) {
|
||||
return PassAction::Replace(new_node);
|
||||
}
|
||||
|
||||
// TODO: does this ever help anything?
|
||||
if bc.cps.intervals().len() > bc.cps.inverted_interval_count() {
|
||||
bc.cps = bc.cps.inverted();
|
||||
bc.invert = !bc.invert;
|
||||
PassAction::Modified
|
||||
} else {
|
||||
PassAction::Keep
|
||||
}
|
||||
}
|
||||
_ => PassAction::Keep,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn optimize(r: &mut Regex) {
|
||||
run_pass(r, &mut simplify_brackets);
|
||||
loop {
|
||||
let mut changed = false;
|
||||
changed |= run_pass(r, &mut decat);
|
||||
if r.flags.icase {
|
||||
changed |= run_pass(r, &mut unfold_icase_chars);
|
||||
}
|
||||
changed |= run_pass(r, &mut unroll_loops);
|
||||
changed |= run_pass(r, &mut promote_1char_loops);
|
||||
#[cfg(not(feature = "utf16"))]
|
||||
{
|
||||
changed |= run_pass(r, &mut form_literal_bytes);
|
||||
}
|
||||
changed |= run_pass(r, &mut remove_empties);
|
||||
changed |= run_pass(r, &mut propagate_early_fails);
|
||||
if !changed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+2078
File diff suppressed because it is too large
Load Diff
Vendored
+501
@@ -0,0 +1,501 @@
|
||||
//! PikeVM regex execution engine
|
||||
|
||||
use crate::api::Match;
|
||||
use crate::bytesearch::charset_contains;
|
||||
use crate::cursor;
|
||||
use crate::cursor::{Backward, Direction, Forward};
|
||||
use crate::exec;
|
||||
use crate::indexing::{AsciiInput, ElementType, InputIndexer, Utf8Input};
|
||||
use crate::insn::{CompiledRegex, Insn, LoopFields, StartPredicate};
|
||||
use crate::matchers;
|
||||
use crate::matchers::CharProperties;
|
||||
use crate::position::PositionType;
|
||||
use crate::scm;
|
||||
use crate::scm::SingleCharMatcher;
|
||||
use crate::types::{GroupData, LoopData};
|
||||
use crate::util::DebugCheckIndex;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{string::String, vec::Vec};
|
||||
use core::ops::Range;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct State<Position: PositionType> {
|
||||
/// Position in the input string.
|
||||
pos: Position,
|
||||
|
||||
/// Offset in the bytecode.
|
||||
ip: usize,
|
||||
|
||||
/// Loop datas.
|
||||
loops: Vec<LoopData<Position>>,
|
||||
|
||||
/// Group datas.
|
||||
groups: Vec<GroupData<Position>>,
|
||||
}
|
||||
|
||||
enum StateMatch<Position: PositionType> {
|
||||
Fail,
|
||||
Continue,
|
||||
Split(State<Position>),
|
||||
Complete,
|
||||
}
|
||||
|
||||
fn run_loop<Position: PositionType>(
|
||||
s: &mut State<Position>,
|
||||
lf: &LoopFields,
|
||||
is_initial_entry: bool,
|
||||
) -> StateMatch<Position> {
|
||||
debug_assert!(lf.max_iters >= lf.min_iters);
|
||||
let ld = &mut s.loops[lf.loop_id as usize];
|
||||
let exit = lf.exit as usize;
|
||||
let skip_ok;
|
||||
let enter_ok;
|
||||
if is_initial_entry {
|
||||
// Entering the loop for the "first" time.
|
||||
ld.iters = 0;
|
||||
enter_ok = lf.max_iters > 0;
|
||||
skip_ok = lf.min_iters == 0;
|
||||
} else {
|
||||
// Note that iters is the number of complete iterations.
|
||||
ld.iters += 1;
|
||||
// We can enter the loop if we have iterated less than the maximum number of
|
||||
// times.
|
||||
enter_ok = ld.iters < lf.max_iters;
|
||||
|
||||
// We can skip the loop if we have iterated at least the minimum number of
|
||||
// times.
|
||||
skip_ok = ld.iters >= lf.min_iters;
|
||||
|
||||
// Check if this iteration was beyond the minimum number of times, and our entry
|
||||
// position is the same as last time (ES6 21.2.2.5.1 note 4).
|
||||
// If so, we matched the empty string and we stop.
|
||||
if ld.iters > lf.min_iters && ld.entry == s.pos {
|
||||
return StateMatch::Fail;
|
||||
}
|
||||
}
|
||||
// Set up our fields as if we are going to enter the loop.
|
||||
ld.entry = s.pos;
|
||||
s.ip += 1;
|
||||
|
||||
if !enter_ok && !skip_ok {
|
||||
StateMatch::Fail
|
||||
} else if !enter_ok {
|
||||
s.ip = exit;
|
||||
StateMatch::Continue
|
||||
} else if !skip_ok {
|
||||
StateMatch::Continue
|
||||
} else {
|
||||
debug_assert!(enter_ok && skip_ok);
|
||||
// We need to split our state.
|
||||
let mut newstate = s.clone();
|
||||
let exit_state = if lf.greedy { s } else { &mut newstate };
|
||||
exit_state.ip = exit;
|
||||
StateMatch::Split(newstate)
|
||||
}
|
||||
}
|
||||
|
||||
fn try_match_state<Input: InputIndexer, Dir: Direction>(
|
||||
re: &CompiledRegex,
|
||||
input: &Input,
|
||||
s: &mut State<Input::Position>,
|
||||
dir: Dir,
|
||||
) -> StateMatch<Input::Position> {
|
||||
macro_rules! nextinsn_or_fail {
|
||||
($e:expr) => {
|
||||
if $e {
|
||||
s.ip += 1;
|
||||
StateMatch::Continue
|
||||
} else {
|
||||
StateMatch::Fail
|
||||
}
|
||||
};
|
||||
}
|
||||
match &re.insns[s.ip] {
|
||||
Insn::Goal => StateMatch::Complete,
|
||||
Insn::JustFail => StateMatch::Fail,
|
||||
&Insn::Char(c) => match cursor::next(input, dir, &mut s.pos) {
|
||||
Some(c2) => nextinsn_or_fail!(c == c2.as_u32()),
|
||||
_ => StateMatch::Fail,
|
||||
},
|
||||
&Insn::CharICase(c) => match cursor::next(input, dir, &mut s.pos) {
|
||||
Some(c2) => {
|
||||
nextinsn_or_fail!(c == c2.as_u32() || input.fold(c2).as_u32() == c)
|
||||
}
|
||||
_ => StateMatch::Fail,
|
||||
},
|
||||
|
||||
Insn::CharSet(v) => match cursor::next(input, dir, &mut s.pos) {
|
||||
Some(c) => nextinsn_or_fail!(charset_contains(v, c.as_u32())),
|
||||
_ => StateMatch::Fail,
|
||||
},
|
||||
|
||||
Insn::ByteSeq1(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq2(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq3(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq4(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq5(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq6(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq7(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq8(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq9(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq10(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq11(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq12(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq13(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq14(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq15(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
Insn::ByteSeq16(v) => nextinsn_or_fail!(cursor::try_match_lit(input, dir, &mut s.pos, v)),
|
||||
|
||||
Insn::StartOfLine { multiline } => {
|
||||
let multiline = *multiline;
|
||||
let matches = match input.peek_left(s.pos) {
|
||||
None => true,
|
||||
Some(c) if multiline && Input::CharProps::is_line_terminator(c) => true,
|
||||
_ => false,
|
||||
};
|
||||
nextinsn_or_fail!(matches)
|
||||
}
|
||||
|
||||
Insn::EndOfLine { multiline } => {
|
||||
let multiline = *multiline;
|
||||
let matches = match input.peek_right(s.pos) {
|
||||
None => true, // we're at the right of the string
|
||||
Some(c) if multiline && Input::CharProps::is_line_terminator(c) => true,
|
||||
_ => false,
|
||||
};
|
||||
nextinsn_or_fail!(matches)
|
||||
}
|
||||
|
||||
Insn::MatchAny => match cursor::next(input, dir, &mut s.pos) {
|
||||
Some(_) => nextinsn_or_fail!(true),
|
||||
_ => StateMatch::Fail,
|
||||
},
|
||||
|
||||
Insn::MatchAnyExceptLineTerminator => match cursor::next(input, dir, &mut s.pos) {
|
||||
Some(c2) => nextinsn_or_fail!(!Input::CharProps::is_line_terminator(c2)),
|
||||
_ => StateMatch::Fail,
|
||||
},
|
||||
|
||||
&Insn::Jump { target } => {
|
||||
s.ip = target as usize;
|
||||
StateMatch::Continue
|
||||
}
|
||||
|
||||
&Insn::Alt { secondary } => {
|
||||
let mut left = s.clone();
|
||||
left.ip += 1;
|
||||
s.ip = secondary as usize;
|
||||
StateMatch::Split(left)
|
||||
}
|
||||
|
||||
&Insn::BeginCaptureGroup(group_idx) => {
|
||||
let group = &mut s.groups[group_idx as usize];
|
||||
if Dir::FORWARD {
|
||||
core::debug_assert!(!group.start_matched(), "Group should not have been entered");
|
||||
group.start = Some(s.pos);
|
||||
} else {
|
||||
core::debug_assert!(!group.end_matched(), "Group should not have been entered");
|
||||
group.end = Some(s.pos);
|
||||
}
|
||||
nextinsn_or_fail!(true)
|
||||
}
|
||||
|
||||
&Insn::EndCaptureGroup(group_idx) => {
|
||||
let group = &mut s.groups[group_idx as usize];
|
||||
if Dir::FORWARD {
|
||||
core::debug_assert!(group.start_matched(), "Group should have been entered");
|
||||
group.end = Some(s.pos);
|
||||
} else {
|
||||
core::debug_assert!(group.end_matched(), "Group should have been exited");
|
||||
group.start = Some(s.pos);
|
||||
}
|
||||
core::debug_assert!(
|
||||
group.end >= group.start,
|
||||
"Exit pos should be after start pos"
|
||||
);
|
||||
nextinsn_or_fail!(true)
|
||||
}
|
||||
|
||||
&Insn::ResetCaptureGroup(group_idx) => {
|
||||
s.groups[group_idx as usize].reset();
|
||||
nextinsn_or_fail!(true)
|
||||
}
|
||||
|
||||
&Insn::BackRef(group_idx) => {
|
||||
let matched;
|
||||
let group = &mut s.groups[group_idx as usize];
|
||||
if let Some(orig_range) = group.as_range() {
|
||||
if re.flags.icase {
|
||||
matched = matchers::backref_icase(input, dir, orig_range, &mut s.pos);
|
||||
} else {
|
||||
matched = matchers::backref(input, dir, orig_range, &mut s.pos)
|
||||
}
|
||||
} else {
|
||||
// This group has not been exited, and therefore the match succeeds
|
||||
// (ES6 21.2.2.9).
|
||||
matched = true;
|
||||
}
|
||||
nextinsn_or_fail!(matched)
|
||||
}
|
||||
|
||||
&Insn::Lookahead {
|
||||
negate,
|
||||
start_group: _,
|
||||
end_group: _,
|
||||
continuation,
|
||||
} => {
|
||||
// Enter into the lookaround's instruction stream.
|
||||
s.ip += 1;
|
||||
let saved_pos = s.pos;
|
||||
let attempt_succeeded =
|
||||
MatchAttempter::<Input>::new(re).try_at_pos(*input, s, Forward::new());
|
||||
let matched = attempt_succeeded != negate;
|
||||
if matched {
|
||||
s.ip = continuation as usize;
|
||||
s.pos = saved_pos;
|
||||
StateMatch::Continue
|
||||
} else {
|
||||
StateMatch::Fail
|
||||
}
|
||||
}
|
||||
|
||||
&Insn::Lookbehind {
|
||||
negate,
|
||||
start_group: _,
|
||||
end_group: _,
|
||||
continuation,
|
||||
} => {
|
||||
// Enter into the lookaround's instruction stream.
|
||||
s.ip += 1;
|
||||
let saved_pos = s.pos;
|
||||
let attempt_succeeded = MatchAttempter::new(re).try_at_pos(*input, s, Backward::new());
|
||||
let matched = attempt_succeeded != negate;
|
||||
if matched {
|
||||
s.ip = continuation as usize;
|
||||
s.pos = saved_pos;
|
||||
StateMatch::Continue
|
||||
} else {
|
||||
StateMatch::Fail
|
||||
}
|
||||
}
|
||||
Insn::EnterLoop(lf) => run_loop(s, lf, true),
|
||||
&Insn::LoopAgain { begin } => {
|
||||
s.ip = begin as usize;
|
||||
match re.insns.iat(s.ip) {
|
||||
Insn::EnterLoop(lf) => run_loop(s, lf, false),
|
||||
_ => panic!("LoopAgain does not point at EnterLoop"),
|
||||
}
|
||||
}
|
||||
Insn::Loop1CharBody { .. } => panic!("Loop1CharBody unimplemented for pikevm"),
|
||||
&Insn::Bracket(idx) => match cursor::next(input, dir, &mut s.pos) {
|
||||
Some(c) => nextinsn_or_fail!(Input::CharProps::bracket(&re.brackets[idx], c)),
|
||||
_ => StateMatch::Fail,
|
||||
},
|
||||
|
||||
Insn::AsciiBracket(bytes) => {
|
||||
nextinsn_or_fail!(scm::MatchByteSet { bytes }.matches(input, dir, &mut s.pos))
|
||||
}
|
||||
&Insn::ByteSet2(bytes) => {
|
||||
nextinsn_or_fail!(scm::MatchByteArraySet(bytes).matches(input, dir, &mut s.pos))
|
||||
}
|
||||
&Insn::ByteSet3(bytes) => {
|
||||
nextinsn_or_fail!(scm::MatchByteArraySet(bytes).matches(input, dir, &mut s.pos))
|
||||
}
|
||||
&Insn::ByteSet4(bytes) => {
|
||||
nextinsn_or_fail!(scm::MatchByteArraySet(bytes).matches(input, dir, &mut s.pos))
|
||||
}
|
||||
|
||||
&Insn::WordBoundary { invert } => {
|
||||
let prev_wordchar = input
|
||||
.peek_left(s.pos)
|
||||
.is_some_and(Input::CharProps::is_word_char);
|
||||
let curr_wordchar = input
|
||||
.peek_right(s.pos)
|
||||
.is_some_and(Input::CharProps::is_word_char);
|
||||
let is_boundary = prev_wordchar != curr_wordchar;
|
||||
nextinsn_or_fail!(is_boundary != invert)
|
||||
}
|
||||
|
||||
&Insn::WordBoundaryUnicodeICase { invert } => {
|
||||
let prev_wordchar = input
|
||||
.peek_left(s.pos)
|
||||
.is_some_and(Input::CharProps::is_word_char_unicode_icase);
|
||||
let curr_wordchar = input
|
||||
.peek_right(s.pos)
|
||||
.is_some_and(Input::CharProps::is_word_char_unicode_icase);
|
||||
let is_boundary = prev_wordchar != curr_wordchar;
|
||||
nextinsn_or_fail!(is_boundary != invert)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn successful_match<Input: InputIndexer>(
|
||||
input: Input,
|
||||
start: Input::Position,
|
||||
state: &State<Input::Position>,
|
||||
group_names: Box<[Box<str>]>,
|
||||
) -> Match {
|
||||
let group_to_offset = |mr: &GroupData<Input::Position>| -> Option<Range<usize>> {
|
||||
mr.as_range().map(|r| Range {
|
||||
start: input.pos_to_offset(r.start),
|
||||
end: input.pos_to_offset(r.end),
|
||||
})
|
||||
};
|
||||
let captures = state.groups.iter().map(group_to_offset).collect();
|
||||
Match {
|
||||
range: input.pos_to_offset(start)..input.pos_to_offset(state.pos),
|
||||
captures,
|
||||
group_names,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct MatchAttempter<'a, Input: InputIndexer> {
|
||||
states: Vec<State<Input::Position>>,
|
||||
re: &'a CompiledRegex,
|
||||
}
|
||||
|
||||
impl<'a, Input: InputIndexer> MatchAttempter<'a, Input> {
|
||||
fn new(re: &'a CompiledRegex) -> Self {
|
||||
Self {
|
||||
states: Vec::new(),
|
||||
re,
|
||||
}
|
||||
}
|
||||
|
||||
fn try_at_pos<Dir: Direction>(
|
||||
&mut self,
|
||||
input: Input,
|
||||
init_state: &mut State<Input::Position>,
|
||||
dir: Dir,
|
||||
) -> bool {
|
||||
debug_assert!(self.states.is_empty(), "Should be no states");
|
||||
self.states.push(init_state.clone());
|
||||
while !self.states.is_empty() {
|
||||
let s = self.states.last_mut().unwrap();
|
||||
match try_match_state(self.re, &input, s, dir) {
|
||||
StateMatch::Fail => {
|
||||
self.states.pop();
|
||||
}
|
||||
StateMatch::Continue => {}
|
||||
StateMatch::Complete => {
|
||||
// Give the successful state to the caller.
|
||||
core::mem::swap(init_state, s);
|
||||
self.states.clear();
|
||||
return true;
|
||||
}
|
||||
StateMatch::Split(newstate) => self.states.push(newstate),
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PikeVMExecutor<'r, Input: InputIndexer> {
|
||||
input: Input,
|
||||
matcher: MatchAttempter<'r, Input>,
|
||||
}
|
||||
|
||||
impl<'r, 't> exec::Executor<'r, 't> for PikeVMExecutor<'r, Utf8Input<'t>> {
|
||||
type AsAscii = PikeVMExecutor<'r, AsciiInput<'t>>;
|
||||
|
||||
fn new(re: &'r CompiledRegex, text: &'t str) -> Self {
|
||||
let input = Utf8Input::new(text, re.flags.unicode);
|
||||
Self {
|
||||
input,
|
||||
matcher: MatchAttempter::new(re),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'r, 't> exec::Executor<'r, 't> for PikeVMExecutor<'r, AsciiInput<'t>> {
|
||||
type AsAscii = PikeVMExecutor<'r, AsciiInput<'t>>;
|
||||
|
||||
fn new(re: &'r CompiledRegex, text: &'t str) -> Self {
|
||||
let input = AsciiInput::new(text, re.flags.unicode);
|
||||
Self {
|
||||
input,
|
||||
matcher: MatchAttempter::new(re),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<Input: InputIndexer> exec::MatchProducer for PikeVMExecutor<'_, Input> {
|
||||
type Position = Input::Position;
|
||||
|
||||
fn initial_position(&self, offset: usize) -> Option<Self::Position> {
|
||||
self.input.try_move_right(self.input.left_end(), offset)
|
||||
}
|
||||
|
||||
fn next_match(
|
||||
&mut self,
|
||||
pos: Self::Position,
|
||||
next_start: &mut Option<Self::Position>,
|
||||
) -> Option<Match> {
|
||||
let re = self.matcher.re;
|
||||
|
||||
// Check if this is an anchored regex - if so, only try matching at the current position
|
||||
if matches!(re.start_pred, StartPredicate::StartAnchored) {
|
||||
let mut state = State {
|
||||
pos,
|
||||
ip: 0,
|
||||
loops: vec![LoopData::new(pos); re.loops as usize],
|
||||
groups: vec![GroupData::new(); re.groups as usize],
|
||||
};
|
||||
if self
|
||||
.matcher
|
||||
.try_at_pos(self.input, &mut state, Forward::new())
|
||||
{
|
||||
let end = state.pos;
|
||||
if end != pos {
|
||||
*next_start = Some(end)
|
||||
} else {
|
||||
*next_start = self.input.next_right_pos(end)
|
||||
}
|
||||
return Some(successful_match(
|
||||
self.input,
|
||||
pos,
|
||||
&state,
|
||||
re.group_names.clone(),
|
||||
));
|
||||
}
|
||||
// Anchored regex failed to match at this position
|
||||
return None;
|
||||
}
|
||||
|
||||
// Standard matching - try at each position
|
||||
// Note the "initial" loop position is ignored. Use whatever is most convenient.
|
||||
let mut state = State {
|
||||
pos,
|
||||
ip: 0,
|
||||
loops: vec![LoopData::new(pos); re.loops as usize],
|
||||
groups: vec![GroupData::new(); re.groups as usize],
|
||||
};
|
||||
loop {
|
||||
let start = state.pos;
|
||||
if self
|
||||
.matcher
|
||||
.try_at_pos(self.input, &mut state, Forward::new())
|
||||
{
|
||||
let end = state.pos;
|
||||
if end != start {
|
||||
*next_start = Some(end)
|
||||
} else {
|
||||
*next_start = self.input.next_right_pos(end)
|
||||
}
|
||||
return Some(successful_match(
|
||||
self.input,
|
||||
start,
|
||||
&state,
|
||||
re.group_names.clone(),
|
||||
));
|
||||
}
|
||||
match self.input.next_right_pos(start) {
|
||||
Some(nextpos) => state.pos = nextpos,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
Vendored
+185
@@ -0,0 +1,185 @@
|
||||
use core::cmp::Eq;
|
||||
use core::fmt;
|
||||
use core::marker::PhantomData;
|
||||
use core::ops;
|
||||
|
||||
/// A trait which references a position in an input string.
|
||||
/// The intent is that this may be satisfied via indexes or pointers.
|
||||
/// Positions must be subtractable, producing usize; they also obey other "pointer arithmetic" ideas.
|
||||
pub trait PositionType:
|
||||
core::fmt::Debug + Copy + Clone + PartialEq + Eq + PartialOrd + Ord
|
||||
where
|
||||
Self: ops::Add<usize, Output = Self>,
|
||||
Self: ops::Sub<usize, Output = Self>,
|
||||
Self: ops::Sub<Self, Output = usize>,
|
||||
Self: ops::AddAssign<usize>,
|
||||
Self: ops::SubAssign<usize>,
|
||||
{
|
||||
}
|
||||
|
||||
/// Choose the preferred position type with this alias.
|
||||
#[cfg(feature = "index-positions")]
|
||||
pub type DefPosition<'a> = IndexPosition<'a>;
|
||||
|
||||
#[cfg(not(feature = "index-positions"))]
|
||||
pub type DefPosition<'a> = RefPosition<'a>;
|
||||
|
||||
/// A simple index-based position.
|
||||
/// It remembers the lifetime of the slice it is tied to.
|
||||
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct IndexPosition<'a>(usize, PhantomData<&'a ()>);
|
||||
|
||||
impl<'a> fmt::Debug for IndexPosition<'a> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_tuple("IndexPosition").field(&self.0).finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl IndexPosition<'_> {
|
||||
/// IndexPosition does not enforce its size.
|
||||
#[inline(always)]
|
||||
pub fn check_size() {}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn new(pos: usize) -> Self {
|
||||
Self(pos, PhantomData)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Add<usize> for IndexPosition<'_> {
|
||||
type Output = Self;
|
||||
|
||||
#[inline(always)]
|
||||
fn add(self, rhs: usize) -> Self::Output {
|
||||
debug_assert!(self.0 + rhs >= self.0, "Overflow");
|
||||
IndexPosition(self.0 + rhs, PhantomData)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::AddAssign<usize> for IndexPosition<'_> {
|
||||
#[inline(always)]
|
||||
fn add_assign(&mut self, rhs: usize) {
|
||||
*self = *self + rhs;
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::SubAssign<usize> for IndexPosition<'_> {
|
||||
#[inline(always)]
|
||||
fn sub_assign(&mut self, rhs: usize) {
|
||||
*self = *self - rhs;
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> ops::Sub<IndexPosition<'a>> for IndexPosition<'a> {
|
||||
type Output = usize;
|
||||
|
||||
#[inline(always)]
|
||||
fn sub(self, rhs: Self) -> Self::Output {
|
||||
debug_assert!(self.0 >= rhs.0, "Underflow");
|
||||
self.0 - rhs.0
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Sub<usize> for IndexPosition<'_> {
|
||||
type Output = Self;
|
||||
|
||||
#[inline(always)]
|
||||
fn sub(self, rhs: usize) -> Self::Output {
|
||||
debug_assert!(self.0 >= rhs, "Underflow");
|
||||
IndexPosition(self.0 - rhs, PhantomData)
|
||||
}
|
||||
}
|
||||
|
||||
impl PositionType for IndexPosition<'_> {}
|
||||
|
||||
/// A reference position holds a reference to a byte and uses pointer arithmetic.
|
||||
/// This must use raw pointers because it must be capable of representing the one-past-the-end value.
|
||||
/// TODO: thread lifetimes through this.
|
||||
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct RefPosition<'a>(core::ptr::NonNull<u8>, PhantomData<&'a ()>);
|
||||
|
||||
impl<'a> fmt::Debug for RefPosition<'a> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let addr = self.0.as_ptr() as usize;
|
||||
f.debug_tuple("RefPosition").field(&addr).finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl RefPosition<'_> {
|
||||
/// The big idea of RefPosition is that `Option<RefPosition>` becomes pointer-sized, by using nullptr as the None value.
|
||||
/// Good candidate for const-panics when stabilized.
|
||||
#[inline(always)]
|
||||
pub fn check_size() {
|
||||
if core::mem::size_of::<Option<Self>>() > core::mem::size_of::<*const u8>() {
|
||||
panic!("Option<RefPosition> should be pointer sized")
|
||||
}
|
||||
}
|
||||
|
||||
/// Access the underlying pointer.
|
||||
#[inline(always)]
|
||||
pub fn ptr(self) -> *const u8 {
|
||||
self.0.as_ptr()
|
||||
}
|
||||
|
||||
/// Construct from a pointer, which must never be null.
|
||||
#[inline(always)]
|
||||
pub fn new(ptr: *const u8) -> Self {
|
||||
debug_assert!(!ptr.is_null(), "Pointer cannot be null");
|
||||
// Annoyingly there's no *const NonNull.
|
||||
let mutp = ptr as *mut u8;
|
||||
let nonnullp = if cfg!(feature = "prohibit-unsafe") {
|
||||
core::ptr::NonNull::new(mutp).expect("Pointer was null")
|
||||
} else {
|
||||
unsafe { core::ptr::NonNull::new_unchecked(mutp) }
|
||||
};
|
||||
Self(nonnullp, PhantomData)
|
||||
}
|
||||
}
|
||||
|
||||
impl PositionType for RefPosition<'_> {}
|
||||
|
||||
impl ops::Add<usize> for RefPosition<'_> {
|
||||
type Output = Self;
|
||||
|
||||
#[inline(always)]
|
||||
fn add(self, rhs: usize) -> Self::Output {
|
||||
Self::new(unsafe { self.ptr().add(rhs) })
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> ops::Sub<RefPosition<'a>> for RefPosition<'a> {
|
||||
type Output = usize;
|
||||
|
||||
#[inline(always)]
|
||||
fn sub(self, rhs: Self) -> Self::Output {
|
||||
debug_assert!(self.0 >= rhs.0, "Underflow");
|
||||
// Note Rust has backwards naming here. The "origin" is self, not the param; the rhs is the offset value.
|
||||
unsafe { self.ptr().offset_from(rhs.ptr()) as usize }
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Sub<usize> for RefPosition<'_> {
|
||||
type Output = Self;
|
||||
|
||||
#[inline(always)]
|
||||
fn sub(self, rhs: usize) -> Self::Output {
|
||||
debug_assert!(self.ptr() as usize >= rhs, "Underflow");
|
||||
Self::new(unsafe { self.ptr().sub(rhs) })
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::AddAssign<usize> for RefPosition<'_> {
|
||||
#[inline(always)]
|
||||
fn add_assign(&mut self, rhs: usize) {
|
||||
*self = *self + rhs;
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::SubAssign<usize> for RefPosition<'_> {
|
||||
#[inline(always)]
|
||||
fn sub_assign(&mut self, rhs: usize) {
|
||||
*self = *self - rhs;
|
||||
}
|
||||
}
|
||||
Vendored
+158
@@ -0,0 +1,158 @@
|
||||
use crate::bytesearch::{ByteArraySet, ByteSet, SmallArraySet, charset_contains};
|
||||
use crate::cursor;
|
||||
use crate::cursor::Direction;
|
||||
use crate::indexing::{ElementType, InputIndexer};
|
||||
use crate::insn::MAX_CHAR_SET_LENGTH;
|
||||
use crate::matchers::CharProperties;
|
||||
use crate::types::BracketContents;
|
||||
|
||||
/// A trait for things that match a single Element.
|
||||
pub trait SingleCharMatcher<Input: InputIndexer, Dir: Direction> {
|
||||
/// \return whether we match the character at the given position, advancing
|
||||
/// the position if so. On a false return, the position is unspecified.
|
||||
fn matches(&self, input: &Input, dir: Dir, pos: &mut Input::Position) -> bool;
|
||||
}
|
||||
|
||||
/// Insn::Char
|
||||
pub struct Char<Input: InputIndexer> {
|
||||
pub c: Input::Element,
|
||||
}
|
||||
impl<Input: InputIndexer, Dir: Direction> SingleCharMatcher<Input, Dir> for Char<Input> {
|
||||
#[inline(always)]
|
||||
fn matches(&self, input: &Input, dir: Dir, pos: &mut Input::Position) -> bool {
|
||||
match cursor::next(input, dir, pos) {
|
||||
Some(c2) => c2 == self.c,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Insn::CharICase
|
||||
pub struct CharICase<Input: InputIndexer> {
|
||||
pub c: Input::Element,
|
||||
}
|
||||
impl<Input: InputIndexer, Dir: Direction> SingleCharMatcher<Input, Dir> for CharICase<Input> {
|
||||
#[inline(always)]
|
||||
fn matches(&self, input: &Input, dir: Dir, pos: &mut Input::Position) -> bool {
|
||||
match cursor::next(input, dir, pos) {
|
||||
Some(c2) => c2 == self.c || input.fold(c2) == self.c,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Insn::CharSet
|
||||
pub struct CharSet<'a> {
|
||||
pub chars: &'a [u32; MAX_CHAR_SET_LENGTH],
|
||||
}
|
||||
|
||||
impl<Input: InputIndexer, Dir: Direction> SingleCharMatcher<Input, Dir> for CharSet<'_> {
|
||||
#[inline(always)]
|
||||
fn matches(&self, input: &Input, dir: Dir, pos: &mut Input::Position) -> bool {
|
||||
match cursor::next(input, dir, pos) {
|
||||
Some(c) => charset_contains(self.chars, c.as_u32()),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Insn::Bracket
|
||||
pub struct Bracket<'a> {
|
||||
pub bc: &'a BracketContents,
|
||||
}
|
||||
|
||||
impl<Input: InputIndexer, Dir: Direction> SingleCharMatcher<Input, Dir> for Bracket<'_> {
|
||||
#[inline(always)]
|
||||
fn matches(&self, input: &Input, dir: Dir, pos: &mut Input::Position) -> bool {
|
||||
match cursor::next(input, dir, pos) {
|
||||
Some(c) => Input::CharProps::bracket(self.bc, c),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Insn::MatchAny
|
||||
pub struct MatchAny {}
|
||||
impl MatchAny {
|
||||
pub fn new() -> Self {
|
||||
Self {}
|
||||
}
|
||||
}
|
||||
impl<Input: InputIndexer, Dir: Direction> SingleCharMatcher<Input, Dir> for MatchAny {
|
||||
#[inline(always)]
|
||||
fn matches(&self, input: &Input, dir: Dir, pos: &mut Input::Position) -> bool {
|
||||
// If there is a character, it counts as a match.
|
||||
cursor::next(input, dir, pos).is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// Insn::MatchAnyExceptLineTerminator
|
||||
pub struct MatchAnyExceptLineTerminator {}
|
||||
impl MatchAnyExceptLineTerminator {
|
||||
pub fn new() -> Self {
|
||||
Self {}
|
||||
}
|
||||
}
|
||||
impl<Input: InputIndexer, Dir: Direction> SingleCharMatcher<Input, Dir>
|
||||
for MatchAnyExceptLineTerminator
|
||||
{
|
||||
#[inline(always)]
|
||||
fn matches(&self, input: &Input, dir: Dir, pos: &mut Input::Position) -> bool {
|
||||
match cursor::next(input, dir, pos) {
|
||||
Some(c2) => !Input::CharProps::is_line_terminator(c2),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Any ByteSet may match a single char.
|
||||
pub struct MatchByteSet<'a, Bytes: ByteSet> {
|
||||
pub bytes: &'a Bytes,
|
||||
}
|
||||
|
||||
impl<Input: InputIndexer, Dir: Direction, Bytes: ByteSet> SingleCharMatcher<Input, Dir>
|
||||
for MatchByteSet<'_, Bytes>
|
||||
{
|
||||
#[inline(always)]
|
||||
fn matches(&self, input: &Input, dir: Dir, pos: &mut Input::Position) -> bool {
|
||||
if Input::CODE_UNITS_ARE_BYTES {
|
||||
// Code units are bytes so we can skip decoding the full element.
|
||||
cursor::next_byte(input, dir, pos).is_some_and(|b| self.bytes.contains(b))
|
||||
} else {
|
||||
// Must decode the full element.
|
||||
cursor::next(input, dir, pos)
|
||||
.and_then(|c| c.as_u32().try_into().ok())
|
||||
.is_some_and(|c| self.bytes.contains(c))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Provide a variant for ByteArraySet where we hold it directly.
|
||||
/// The arrays are small and so we don't want to indirect through a pointer.
|
||||
pub struct MatchByteArraySet<ArraySet: SmallArraySet>(pub ByteArraySet<ArraySet>);
|
||||
|
||||
impl<Input: InputIndexer, Dir: Direction, ArraySet: SmallArraySet> SingleCharMatcher<Input, Dir>
|
||||
for MatchByteArraySet<ArraySet>
|
||||
{
|
||||
#[inline(always)]
|
||||
fn matches(&self, input: &Input, dir: Dir, pos: &mut Input::Position) -> bool {
|
||||
if let Some(b) = cursor::next_byte(input, dir, pos) {
|
||||
self.0.contains(b)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A ByteSeq of length <= 4 may match a single char.
|
||||
pub struct MatchByteSeq<'a, const N: usize>(pub &'a [u8; N]);
|
||||
|
||||
impl<const N: usize, Input: InputIndexer, Dir: Direction> SingleCharMatcher<Input, Dir>
|
||||
for MatchByteSeq<'_, N>
|
||||
{
|
||||
#[inline(always)]
|
||||
fn matches(&self, input: &Input, dir: Dir, pos: &mut Input::Position) -> bool {
|
||||
debug_assert!(N <= 4, "This looks like it could match more than one char");
|
||||
cursor::try_match_lit(input, dir, pos, self.0)
|
||||
}
|
||||
}
|
||||
+223
@@ -0,0 +1,223 @@
|
||||
//! Support for quickly finding potential match locations.
|
||||
use crate::bytesearch::ByteBitmap;
|
||||
use crate::codepointset;
|
||||
use crate::insn::StartPredicate;
|
||||
use crate::ir;
|
||||
use crate::ir::Node;
|
||||
use crate::util::{add_utf8_first_bytes_to_bitmap, utf8_first_byte};
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::{boxed::Box, vec::Vec};
|
||||
use memchr::memmem;
|
||||
|
||||
/// Check if a node is anchored to the start of the line/string.
|
||||
/// Returns true if the node begins with a StartOfLine anchor.
|
||||
fn is_start_anchored(n: &Node) -> bool {
|
||||
match n {
|
||||
Node::Anchor {
|
||||
anchor_type: ir::AnchorType::StartOfLine,
|
||||
multiline,
|
||||
} => !multiline,
|
||||
Node::Cat(nodes) => {
|
||||
// For concatenation, check if the first node is start-anchored
|
||||
nodes.first().is_some_and(is_start_anchored)
|
||||
}
|
||||
Node::CaptureGroup { contents, .. } => is_start_anchored(contents),
|
||||
// Other nodes are not anchored
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert the code point set to a first-byte bitmap.
|
||||
/// That is, make a list of all of the possible first bytes of every contained
|
||||
/// code point, and store that in a bitmap.
|
||||
fn cps_to_first_byte_bitmap(input: &codepointset::CodePointSet) -> Box<ByteBitmap> {
|
||||
let mut bitmap = Box::<ByteBitmap>::default();
|
||||
for iv in input.intervals() {
|
||||
add_utf8_first_bytes_to_bitmap(*iv, &mut bitmap);
|
||||
}
|
||||
bitmap
|
||||
}
|
||||
|
||||
/// The "IR" for a start predicate.
|
||||
enum AbstractStartPredicate {
|
||||
/// No predicate.
|
||||
Arbitrary,
|
||||
|
||||
/// Sequence of non-empty bytes.
|
||||
Sequence(Vec<u8>),
|
||||
|
||||
/// Set of bytes.
|
||||
Set(Box<ByteBitmap>),
|
||||
}
|
||||
|
||||
impl AbstractStartPredicate {
|
||||
/// \return the disjunction of two predicates.
|
||||
/// That is, a predicate that matches x OR y.
|
||||
fn disjunction(x: Self, y: Self) -> Self {
|
||||
match (x, y) {
|
||||
(Self::Arbitrary, _) => Self::Arbitrary,
|
||||
(_, Self::Arbitrary) => Self::Arbitrary,
|
||||
|
||||
(Self::Sequence(s1), Self::Sequence(s2)) => {
|
||||
// Compute the length of the shared prefix.
|
||||
let shared_len = s1.iter().zip(s2.iter()).take_while(|(a, b)| a == b).count();
|
||||
debug_assert!(s1[..shared_len] == s2[..shared_len]);
|
||||
if shared_len > 0 {
|
||||
// Use the shared prefix.
|
||||
Self::Sequence(s1[..shared_len].to_vec())
|
||||
} else {
|
||||
// Use a set of their first byte.
|
||||
Self::Set(Box::new(ByteBitmap::new(&[s1[0], s2[0]])))
|
||||
}
|
||||
}
|
||||
|
||||
(Self::Set(mut s1), Self::Set(s2)) => {
|
||||
s1.bitor(s2.as_ref());
|
||||
Self::Set(s1)
|
||||
}
|
||||
|
||||
(Self::Set(mut s1), Self::Sequence(s2)) => {
|
||||
// Add first byte to set.
|
||||
s1.set(s2[0]);
|
||||
Self::Set(s1)
|
||||
}
|
||||
|
||||
(Self::Sequence(s1), Self::Set(mut s2)) => {
|
||||
s2.set(s1[0]);
|
||||
Self::Set(s2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve ourselves to a concrete start predicate.
|
||||
fn resolve_to_insn(self) -> StartPredicate {
|
||||
match self {
|
||||
Self::Arbitrary => StartPredicate::Arbitrary,
|
||||
Self::Sequence(vals) => match vals.len() {
|
||||
0 => StartPredicate::Arbitrary,
|
||||
1 => StartPredicate::ByteSet1([vals[0]]),
|
||||
_ => StartPredicate::ByteSeq(Box::new(memmem::Finder::new(&vals).into_owned())),
|
||||
},
|
||||
Self::Set(bm) => match bm.count_bits() {
|
||||
0 => StartPredicate::Arbitrary,
|
||||
1 => StartPredicate::ByteSet1(bm.as_array()),
|
||||
2 => StartPredicate::ByteSet2(bm.as_array()),
|
||||
3 => StartPredicate::ByteSet3(bm.as_array()),
|
||||
_ => StartPredicate::ByteBracket(*bm),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute any start-predicate for a node..
|
||||
/// If this returns None, then the instruction is conceptually zero-width (e.g.
|
||||
/// lookahead assertion) and does not contribute to the predicate.
|
||||
/// If this returns StartPredicate::Arbitrary, then there is no predicate.
|
||||
fn compute_start_predicate(n: &Node) -> Option<AbstractStartPredicate> {
|
||||
let arbitrary = Some(AbstractStartPredicate::Arbitrary);
|
||||
match n {
|
||||
Node::ByteSequence(bytevec) => Some(AbstractStartPredicate::Sequence(bytevec.clone())),
|
||||
Node::ByteSet(bytes) => Some(AbstractStartPredicate::Set(Box::new(ByteBitmap::new(
|
||||
bytes,
|
||||
)))),
|
||||
|
||||
Node::Empty => arbitrary,
|
||||
Node::Goal => arbitrary,
|
||||
Node::BackRef(..) => arbitrary,
|
||||
|
||||
Node::CharSet(chars) => {
|
||||
// Pick the first bytes out.
|
||||
let bytes = chars
|
||||
.iter()
|
||||
.map(|&c| utf8_first_byte(c))
|
||||
.collect::<Vec<_>>();
|
||||
Some(AbstractStartPredicate::Set(Box::new(ByteBitmap::new(
|
||||
&bytes,
|
||||
))))
|
||||
}
|
||||
|
||||
// We assume that most char nodes have been optimized to ByteSeq or AnyBytes2, so skip
|
||||
// these.
|
||||
// TODO: we could support icase through bitmap of de-folded first bytes.
|
||||
Node::Char { .. } => arbitrary,
|
||||
|
||||
// Cats return the first non-None value, if any.
|
||||
Node::Cat(nodes) => nodes.iter().filter_map(compute_start_predicate).next(),
|
||||
|
||||
// MatchAny (aka .) is too common to do a fast prefix search for.
|
||||
Node::MatchAny => arbitrary,
|
||||
|
||||
// MatchAnyExceptLineTerminator (aka .) is too common to do a fast prefix search for.
|
||||
Node::MatchAnyExceptLineTerminator => arbitrary,
|
||||
|
||||
// TODO: can probably exploit some of these.
|
||||
Node::Anchor { .. } => arbitrary,
|
||||
Node::WordBoundary { .. } => arbitrary,
|
||||
|
||||
// Capture groups delegate to their contents.
|
||||
Node::CaptureGroup { contents, .. } => compute_start_predicate(contents),
|
||||
|
||||
// Zero-width assertions are one of the few instructions that impose no start predicate.
|
||||
Node::LookaroundAssertion { .. } => None,
|
||||
|
||||
Node::Loop { loopee, quant, .. } => {
|
||||
// TODO: we could try to join two predicates if the loop were optional.
|
||||
if quant.min > 0 {
|
||||
compute_start_predicate(loopee)
|
||||
} else {
|
||||
arbitrary
|
||||
}
|
||||
}
|
||||
|
||||
Node::Loop1CharBody { loopee, quant } => {
|
||||
// TODO: we could try to join two predicates if the loop were optional.
|
||||
if quant.min > 0 {
|
||||
compute_start_predicate(loopee)
|
||||
} else {
|
||||
arbitrary
|
||||
}
|
||||
}
|
||||
|
||||
// This one is interesting - we compute the disjunction of the predicates of our two arms.
|
||||
Node::Alt(left, right) => {
|
||||
if let (Some(x), Some(y)) = (
|
||||
compute_start_predicate(left),
|
||||
compute_start_predicate(right),
|
||||
) {
|
||||
Some(AbstractStartPredicate::disjunction(x, y))
|
||||
} else {
|
||||
// This indicates that one of our branches could match the empty string.
|
||||
arbitrary
|
||||
}
|
||||
}
|
||||
|
||||
// Brackets get a bitmap.
|
||||
Node::Bracket(bc) => {
|
||||
// If our bracket is inverted, construct the set of code points not contained.
|
||||
let storage;
|
||||
let cps = if bc.invert {
|
||||
storage = bc.cps.inverted();
|
||||
&storage
|
||||
} else {
|
||||
&bc.cps
|
||||
};
|
||||
let bitmap = cps_to_first_byte_bitmap(cps);
|
||||
Some(AbstractStartPredicate::Set(bitmap))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// \return the start predicate for a Regex.
|
||||
pub fn predicate_for_re(re: &ir::Regex) -> StartPredicate {
|
||||
// Check if the regex is anchored to the start - if so, we can optimize
|
||||
// by avoiding string searching entirely. However, only do this when
|
||||
// multiline mode is disabled, since in multiline mode ^ can match
|
||||
// at the beginning of any line, not just the string start.
|
||||
if is_start_anchored(&re.node) && !re.flags.multiline {
|
||||
return StartPredicate::StartAnchored;
|
||||
}
|
||||
|
||||
compute_start_predicate(&re.node)
|
||||
.unwrap_or(AbstractStartPredicate::Arbitrary)
|
||||
.resolve_to_insn()
|
||||
}
|
||||
Vendored
+98
@@ -0,0 +1,98 @@
|
||||
use crate::codepointset::CodePointSet;
|
||||
use crate::position::PositionType;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::string::String;
|
||||
use core::ops;
|
||||
|
||||
/// A group index is u16.
|
||||
/// CaptureGroupID 0 corresponds to the first capture group.
|
||||
pub type CaptureGroupID = u16;
|
||||
|
||||
/// The name of a named capture group.
|
||||
pub type CaptureGroupName = String;
|
||||
|
||||
/// The maximum number of capture groups supported.
|
||||
pub const MAX_CAPTURE_GROUPS: usize = 65535;
|
||||
|
||||
/// The maximum number of loops supported.
|
||||
pub const MAX_LOOPS: usize = 65535;
|
||||
pub type LoopID = u16;
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub enum CharacterClassType {
|
||||
Digits,
|
||||
Spaces,
|
||||
Words,
|
||||
}
|
||||
|
||||
/// The stuff in a bracket.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct BracketContents {
|
||||
pub invert: bool,
|
||||
pub cps: CodePointSet,
|
||||
}
|
||||
|
||||
impl BracketContents {
|
||||
// Return true if the bracket is empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
match self.invert {
|
||||
false => self.cps.is_empty(),
|
||||
true => self.cps.contains_all_codepoints(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An instruction pointer.
|
||||
pub type IP = usize;
|
||||
|
||||
/// Representation of a loop.
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct LoopData<Position: PositionType> {
|
||||
pub iters: usize,
|
||||
pub entry: Position,
|
||||
}
|
||||
|
||||
impl<Position: PositionType> LoopData<Position> {
|
||||
pub fn new(entry: Position) -> LoopData<Position> {
|
||||
LoopData { iters: 0, entry }
|
||||
}
|
||||
}
|
||||
|
||||
/// Representation of a capture group.
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct GroupData<Position: PositionType> {
|
||||
pub start: Option<Position>,
|
||||
pub end: Option<Position>,
|
||||
}
|
||||
|
||||
impl<Position: PositionType> GroupData<Position> {
|
||||
pub fn new() -> GroupData<Position> {
|
||||
GroupData {
|
||||
start: None,
|
||||
end: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_matched(&self) -> bool {
|
||||
self.start.is_some()
|
||||
}
|
||||
|
||||
pub fn end_matched(&self) -> bool {
|
||||
self.end.is_some()
|
||||
}
|
||||
|
||||
pub fn as_range(&self) -> Option<ops::Range<Position>> {
|
||||
// Note: we may have only start_matched (if forwards) or end_matched (if
|
||||
// backwards) set.
|
||||
match (self.start, self.end) {
|
||||
(Some(start), Some(end)) => Some(ops::Range { start, end }),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset the group to "not entered."
|
||||
pub fn reset(&mut self) {
|
||||
self.start = None;
|
||||
self.end = None;
|
||||
}
|
||||
}
|
||||
Vendored
+581
@@ -0,0 +1,581 @@
|
||||
use crate::codepointset::{CodePointSet, Interval};
|
||||
use crate::unicodetables::{
|
||||
FOLDS, TO_UPPERCASE, binary_property_ranges, general_category_property_value_ranges,
|
||||
script_extensions_value_ranges, script_value_ranges, string_property_sets,
|
||||
unicode_property_binary_from_str, unicode_property_value_general_category_from_str,
|
||||
unicode_property_value_script_from_str, unicode_string_property_from_str,
|
||||
};
|
||||
use crate::util::SliceHelp;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
use core::cmp::Ordering;
|
||||
|
||||
// CodePointRange packs a code point and a length together into a u32.
|
||||
// We currently do not need to store any information about code points in plane 16 (U+100000),
|
||||
// which are private use, so we only need 20 bits of code point storage;
|
||||
// the remaining 12 can be the length.
|
||||
// The length is stored with a bias of -1, so the last codepoint may be obtained by adding the "length" and the first code point.
|
||||
const CODE_POINT_BITS: u32 = 20;
|
||||
const LENGTH_BITS: u32 = 32 - CODE_POINT_BITS;
|
||||
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub struct CodePointRange(u32);
|
||||
|
||||
// This will trigger an error in const functions if $x is false.
|
||||
macro_rules! const_assert_true {
|
||||
($x:expr $(,)*) => {
|
||||
[()][!$x as usize];
|
||||
};
|
||||
}
|
||||
|
||||
impl CodePointRange {
|
||||
#[inline(always)]
|
||||
pub const fn from(start: u32, len: u32) -> Self {
|
||||
const_assert_true!(start < (1 << CODE_POINT_BITS));
|
||||
const_assert_true!(len > 0 && len <= (1 << LENGTH_BITS));
|
||||
const_assert_true!((start + len - 1) < ((1 << CODE_POINT_BITS) - 1));
|
||||
CodePointRange((start << LENGTH_BITS) | (len - 1))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
const fn len_minus_1(self) -> u32 {
|
||||
self.0 & ((1 << LENGTH_BITS) - 1)
|
||||
}
|
||||
|
||||
// \return the first codepoint in the range.
|
||||
#[inline(always)]
|
||||
pub const fn first(self) -> u32 {
|
||||
self.0 >> LENGTH_BITS
|
||||
}
|
||||
|
||||
// \return the last codepoint in the range.
|
||||
#[inline(always)]
|
||||
pub const fn last(self) -> u32 {
|
||||
self.first() + self.len_minus_1()
|
||||
}
|
||||
}
|
||||
|
||||
// The "extra" field contains a predicate mask in the low bits and a signed delta amount in the high bits.
|
||||
// A code point only transforms if its difference from the range base is 0 once masked.
|
||||
const PREDICATE_MASK_BITS: u32 = 4;
|
||||
|
||||
pub(crate) struct FoldRange {
|
||||
/// The range of codepoints.
|
||||
pub(crate) range: CodePointRange,
|
||||
|
||||
/// Combination of the signed delta amount and predicate mask.
|
||||
pub(crate) extra: i32,
|
||||
}
|
||||
|
||||
impl FoldRange {
|
||||
#[inline(always)]
|
||||
pub const fn from(start: u32, length: u32, delta: i32, modulo: u8) -> Self {
|
||||
const_assert_true!(modulo.is_power_of_two());
|
||||
let mask = (modulo - 1) as i32;
|
||||
const_assert_true!(mask < (1 << PREDICATE_MASK_BITS));
|
||||
const_assert_true!(((delta << PREDICATE_MASK_BITS) >> PREDICATE_MASK_BITS) == delta);
|
||||
let extra = mask | (delta << PREDICATE_MASK_BITS);
|
||||
FoldRange {
|
||||
range: CodePointRange::from(start, length),
|
||||
extra,
|
||||
}
|
||||
}
|
||||
#[inline(always)]
|
||||
fn first(&self) -> u32 {
|
||||
self.range.first()
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn last(&self) -> u32 {
|
||||
self.range.last()
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn delta(&self) -> i32 {
|
||||
self.extra >> PREDICATE_MASK_BITS
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn predicate_mask(&self) -> u32 {
|
||||
(self.extra as u32) & ((1 << PREDICATE_MASK_BITS) - 1)
|
||||
}
|
||||
|
||||
fn add_delta(&self, cu: u32) -> u32 {
|
||||
let cs = (cu as i32) + self.delta();
|
||||
core::debug_assert!(0 <= cs && cs <= 0x10FFFF);
|
||||
cs as u32
|
||||
}
|
||||
|
||||
/// \return the Interval of transformed-to code points.
|
||||
fn transformed_to(&self) -> Interval {
|
||||
Interval {
|
||||
first: self.add_delta(self.first()),
|
||||
last: self.add_delta(self.last()),
|
||||
}
|
||||
}
|
||||
|
||||
/// \return the Interval of transformed-from code points.
|
||||
fn transformed_from(&self) -> Interval {
|
||||
Interval {
|
||||
first: self.first(),
|
||||
last: self.last(),
|
||||
}
|
||||
}
|
||||
|
||||
fn can_apply(&self, cu: u32) -> bool {
|
||||
self.transformed_from().contains(cu)
|
||||
}
|
||||
|
||||
fn apply(&self, cu: u32) -> u32 {
|
||||
debug_assert!(self.can_apply(cu), "Cannot apply to this code point");
|
||||
let offset = cu - self.first();
|
||||
if (offset & self.predicate_mask()) == 0 {
|
||||
self.add_delta(cu)
|
||||
} else {
|
||||
cu
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Implements the `Canonicalize` method from the [spec].
|
||||
///
|
||||
/// [spec]: https://tc39.es/ecma262/#sec-runtime-semantics-canonicalize-ch
|
||||
pub(crate) fn fold_code_point(cu: u32, unicode: bool) -> u32 {
|
||||
if unicode {
|
||||
return fold(cu);
|
||||
}
|
||||
uppercase(cu)
|
||||
}
|
||||
|
||||
pub fn fold(cu: u32) -> u32 {
|
||||
let searched = FOLDS.binary_search_by(|fr| {
|
||||
if fr.first() > cu {
|
||||
Ordering::Greater
|
||||
} else if fr.last() < cu {
|
||||
Ordering::Less
|
||||
} else {
|
||||
Ordering::Equal
|
||||
}
|
||||
});
|
||||
if let Ok(index) = searched {
|
||||
let fr: &FoldRange = if cfg!(feature = "prohibit-unsafe") {
|
||||
unsafe { FOLDS.get_unchecked(index) }
|
||||
} else {
|
||||
FOLDS.get(index).expect("Invalid index")
|
||||
};
|
||||
fr.apply(cu)
|
||||
} else {
|
||||
cu
|
||||
}
|
||||
}
|
||||
|
||||
fn uppercase(cu: u32) -> u32 {
|
||||
let searched = TO_UPPERCASE.binary_search_by(|fr| {
|
||||
if fr.first() > cu {
|
||||
Ordering::Greater
|
||||
} else if fr.last() < cu {
|
||||
Ordering::Less
|
||||
} else {
|
||||
Ordering::Equal
|
||||
}
|
||||
});
|
||||
if let Ok(index) = searched {
|
||||
let fr: &FoldRange = if cfg!(feature = "prohibit-unsafe") {
|
||||
unsafe { TO_UPPERCASE.get_unchecked(index) }
|
||||
} else {
|
||||
TO_UPPERCASE.get(index).expect("Invalid index")
|
||||
};
|
||||
fr.apply(cu)
|
||||
} else {
|
||||
cu
|
||||
}
|
||||
}
|
||||
|
||||
// Add all folded characters in the given interval to the given code point set.
|
||||
// This skips characters which fold to themselves.
|
||||
fn fold_interval(iv: Interval, recv: &mut CodePointSet) {
|
||||
let overlaps = FOLDS.equal_range_by(|tr| {
|
||||
if tr.first() > iv.last {
|
||||
Ordering::Greater
|
||||
} else if tr.last() < iv.first {
|
||||
Ordering::Less
|
||||
} else {
|
||||
Ordering::Equal
|
||||
}
|
||||
});
|
||||
for fr in &FOLDS[overlaps] {
|
||||
debug_assert!(
|
||||
fr.transformed_from().overlaps(iv),
|
||||
"Interval does not overlap transform"
|
||||
);
|
||||
// Find the (inclusive) range of our interval that this transform covers.
|
||||
let first_trans = core::cmp::max(fr.first(), iv.first);
|
||||
let last_trans = core::cmp::min(fr.last(), iv.last);
|
||||
|
||||
let modulo = fr.predicate_mask() + 1;
|
||||
if modulo == 1 {
|
||||
// Optimization: when modulo is 1, every character in range gets transformed
|
||||
for cu in first_trans..(last_trans + 1) {
|
||||
let cs = fr.add_delta(cu);
|
||||
if cs != cu {
|
||||
recv.add_one(cs);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Optimization: walk by modulo amount instead of checking every character
|
||||
let offset_start = first_trans - fr.first();
|
||||
let start_aligned = first_trans + ((modulo - (offset_start % modulo)) % modulo);
|
||||
|
||||
let mut cu = start_aligned;
|
||||
while cu <= last_trans {
|
||||
let cs = fr.add_delta(cu);
|
||||
recv.add_one(cs);
|
||||
cu += modulo;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Find all characters that fold into the given interval and add them to the given code point set.
|
||||
/// This skips characters which fold to themselves.
|
||||
fn unfold_interval(iv: Interval, recv: &mut CodePointSet) {
|
||||
// Note: We still need to check all ranges because the relationship between
|
||||
// transformed_from and transformed_to intervals can be complex
|
||||
for tr in FOLDS.iter() {
|
||||
if !iv.overlaps(tr.transformed_to()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let modulo = tr.predicate_mask() + 1;
|
||||
let first_source = tr.first();
|
||||
let last_source = tr.last();
|
||||
|
||||
let mut process_cp = |cp| {
|
||||
let tcp = tr.apply(cp);
|
||||
if tcp != cp && iv.contains(tcp) {
|
||||
recv.add_one(cp);
|
||||
}
|
||||
};
|
||||
|
||||
if modulo == 1 {
|
||||
// Optimization: when modulo is 1, every character in range gets transformed
|
||||
for cp in first_source..(last_source + 1) {
|
||||
process_cp(cp);
|
||||
}
|
||||
} else {
|
||||
// Walk by modulo amount instead of checking every character
|
||||
let mut cp = first_source;
|
||||
while cp <= last_source {
|
||||
process_cp(cp);
|
||||
cp += modulo;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// \return all the characters which fold to c's fold.
|
||||
/// This is a linear search across all ranges.
|
||||
/// The result always contains c.
|
||||
pub fn unfold_char(c: u32) -> Vec<u32> {
|
||||
let mut res = vec![c];
|
||||
let fcp = fold(c);
|
||||
if fcp != c {
|
||||
res.push(fcp);
|
||||
}
|
||||
// TODO: optimize ASCII case.
|
||||
for tr in FOLDS.iter() {
|
||||
if !tr.transformed_to().contains(fcp) {
|
||||
continue;
|
||||
}
|
||||
for cp in tr.transformed_from().codepoints() {
|
||||
// TODO: this can be optimized.
|
||||
let tcp = tr.apply(cp);
|
||||
if tcp == fcp {
|
||||
res.push(cp);
|
||||
}
|
||||
}
|
||||
}
|
||||
res.sort_unstable();
|
||||
res.dedup();
|
||||
res
|
||||
}
|
||||
|
||||
pub(crate) fn unfold_uppercase_char(c: u32) -> Vec<u32> {
|
||||
let mut res = vec![c];
|
||||
let fcp = uppercase(c);
|
||||
if fcp != c {
|
||||
res.push(fcp);
|
||||
}
|
||||
for tr in TO_UPPERCASE.iter() {
|
||||
if !tr.transformed_to().contains(fcp) {
|
||||
continue;
|
||||
}
|
||||
for cp in tr.transformed_from().codepoints() {
|
||||
let tcp = tr.apply(cp);
|
||||
if tcp == fcp {
|
||||
res.push(cp);
|
||||
}
|
||||
}
|
||||
}
|
||||
res.sort_unstable();
|
||||
res.dedup();
|
||||
res
|
||||
}
|
||||
|
||||
// Fold every character in \p input, then find all the prefolds.
|
||||
pub fn add_icase_code_points(mut input: CodePointSet) -> CodePointSet {
|
||||
let mut folded = input.clone();
|
||||
for iv in input.intervals() {
|
||||
fold_interval(*iv, &mut folded)
|
||||
}
|
||||
|
||||
// Reuse input storage.
|
||||
input.clone_from(&folded);
|
||||
for iv in folded.intervals() {
|
||||
unfold_interval(*iv, &mut input);
|
||||
}
|
||||
input
|
||||
}
|
||||
|
||||
pub(crate) enum PropertyEscapeKind {
|
||||
CharacterClass(&'static [Interval]),
|
||||
StringSet(&'static [&'static [u32]]),
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub(crate) enum UnicodePropertyName {
|
||||
GeneralCategory,
|
||||
Script,
|
||||
ScriptExtensions,
|
||||
}
|
||||
|
||||
pub(crate) fn unicode_property_name_from_str(s: &str) -> Option<UnicodePropertyName> {
|
||||
use UnicodePropertyName::*;
|
||||
|
||||
match s {
|
||||
"General_Category" | "gc" => Some(GeneralCategory),
|
||||
"Script" | "sc" => Some(Script),
|
||||
"Script_Extensions" | "scx" => Some(ScriptExtensions),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn unicode_property_from_str(
|
||||
s: &str,
|
||||
name: Option<UnicodePropertyName>,
|
||||
unicode_sets: bool,
|
||||
) -> Option<PropertyEscapeKind> {
|
||||
match name {
|
||||
Some(UnicodePropertyName::GeneralCategory) => Some(PropertyEscapeKind::CharacterClass(
|
||||
general_category_property_value_ranges(
|
||||
&unicode_property_value_general_category_from_str(s)?,
|
||||
),
|
||||
)),
|
||||
Some(UnicodePropertyName::Script) => Some(PropertyEscapeKind::CharacterClass(
|
||||
script_value_ranges(&unicode_property_value_script_from_str(s)?),
|
||||
)),
|
||||
Some(UnicodePropertyName::ScriptExtensions) => Some(PropertyEscapeKind::CharacterClass(
|
||||
script_extensions_value_ranges(&unicode_property_value_script_from_str(s)?),
|
||||
)),
|
||||
None => {
|
||||
if let Some(value) = unicode_property_binary_from_str(s) {
|
||||
return Some(PropertyEscapeKind::CharacterClass(binary_property_ranges(
|
||||
&value,
|
||||
)));
|
||||
}
|
||||
if unicode_sets && let Some(value) = unicode_string_property_from_str(s) {
|
||||
return Some(PropertyEscapeKind::StringSet(string_property_sets(&value)));
|
||||
}
|
||||
Some(PropertyEscapeKind::CharacterClass(
|
||||
general_category_property_value_ranges(
|
||||
&unicode_property_value_general_category_from_str(s)?,
|
||||
),
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
// Map from folded char to the chars that folded to it.
|
||||
// If an entry is missing, it means either nothing folds to the char,
|
||||
// or it folds exclusively to itself; this can be determined by comparing
|
||||
// the char to its fold.
|
||||
fn get_unfold_map() -> HashMap<u32, Vec<u32>> {
|
||||
let mut unfold_map: HashMap<u32, Vec<u32>> = HashMap::new();
|
||||
for c in 0..=0x10FFFF {
|
||||
let fc = fold(c);
|
||||
if fc != c {
|
||||
unfold_map.entry(fc).or_default().push(c);
|
||||
}
|
||||
}
|
||||
|
||||
// We neglected self-folds - add them now, but only for entries
|
||||
// where something else folds to it, else our map would be quite large.
|
||||
// Also sort them all.
|
||||
for (&k, v) in unfold_map.iter_mut() {
|
||||
assert_eq!(k, fold(k), "folds should be idempotent");
|
||||
v.push(k);
|
||||
v.sort_unstable();
|
||||
}
|
||||
unfold_map
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_folds() {
|
||||
for c in 0..0x41 {
|
||||
assert_eq!(fold(c), c);
|
||||
}
|
||||
for c in 0x41..=0x5A {
|
||||
assert_eq!(fold(c), c + 0x20);
|
||||
}
|
||||
assert_eq!(fold(0xB5), 0x3BC);
|
||||
assert_eq!(fold(0xC0), 0xE0);
|
||||
|
||||
assert_eq!(fold(0x1B8), 0x1B9);
|
||||
assert_eq!(fold(0x1B9), 0x1B9);
|
||||
assert_eq!(fold(0x1BA), 0x1BA);
|
||||
assert_eq!(fold(0x1BB), 0x1BB);
|
||||
assert_eq!(fold(0x1BC), 0x1BD);
|
||||
assert_eq!(fold(0x1BD), 0x1BD);
|
||||
|
||||
for c in 0x1F8..0x21F {
|
||||
if c % 2 == 0 {
|
||||
assert_eq!(fold(c), c + 1);
|
||||
} else {
|
||||
assert_eq!(fold(c), c);
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(fold(0x37F), 0x3F3);
|
||||
assert_eq!(fold(0x380), 0x380);
|
||||
assert_eq!(fold(0x16E40), 0x16E60);
|
||||
assert_eq!(fold(0x16E41), 0x16E61);
|
||||
assert_eq!(fold(0x16E42), 0x16E62);
|
||||
assert_eq!(fold(0x1E900), 0x1E922);
|
||||
assert_eq!(fold(0x1E901), 0x1E923);
|
||||
for c in 0xF0000..=0x10FFFF {
|
||||
assert_eq!(fold(c), c);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fold_idempotent() {
|
||||
for c in 0..=0x10FFFF {
|
||||
let fc = fold(c);
|
||||
let ffc = fold(fc);
|
||||
assert_eq!(ffc, fc);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unfolds_refold() {
|
||||
for c in 0..=0x10FFFF {
|
||||
let fc = fold(c);
|
||||
let unfolds = unfold_char(c);
|
||||
for uc in unfolds {
|
||||
assert_eq!(fold(uc), fc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unfold_chars() {
|
||||
let unfold_map = get_unfold_map();
|
||||
for c in 0..=0x10FFFF {
|
||||
let mut unfolded = unfold_char(c);
|
||||
unfolded.sort_unstable();
|
||||
let fc = fold(c);
|
||||
if let Some(expected) = unfold_map.get(&fc) {
|
||||
// Explicit list of unfolds.
|
||||
assert_eq!(&unfolded, expected);
|
||||
} else {
|
||||
// No entry in our testing unfold map: that means that either the
|
||||
// character folds to itself and nothing else does, or the character
|
||||
// folds to a different character - but that different character
|
||||
// should fold to itself (folding is idempotent) so we should always
|
||||
// have multiple characters in that case. Therefore we expect this
|
||||
// character's unfolds to be itself exclusively.
|
||||
assert_eq!(&unfolded, &[c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_icase_code_points() {
|
||||
let unfold_map = get_unfold_map();
|
||||
let locs = [
|
||||
0x0, 0x42, 0x100, 0xdeba, 0x11419, 0x278f8, 0x2e000, 0x35df7, 0x462d6, 0x4bc29,
|
||||
0x4f4c0, 0x58a9b, 0x5bafc, 0x62383, 0x66d60, 0x6974a, 0x77628, 0x87804, 0x9262b,
|
||||
0x931e4, 0xaa08c, 0xad7a8, 0xca6b0, 0xcce27, 0xcd897, 0xcf5e7, 0xe2802, 0xe561b,
|
||||
0xe5f43, 0xf4339, 0xfb78c, 0xfc5ee, 0x104fa9, 0x10e402, 0x10e6cf, 0x10FFFF,
|
||||
];
|
||||
for (idx, &first) in locs.iter().enumerate() {
|
||||
// Keep a running set of the unfolded code points we expect to be in the
|
||||
// range [first, last].
|
||||
let mut expected = CodePointSet::default();
|
||||
let mut from = first;
|
||||
for &last in &locs[idx..] {
|
||||
// Add both folded and unfolded characters to expected.
|
||||
for c in from..=last {
|
||||
let fc = fold(c);
|
||||
if let Some(unfolded) = unfold_map.get(&fc) {
|
||||
// Some nontrival set of characters fold to fc.
|
||||
for &ufc in unfolded {
|
||||
expected.add_one(ufc);
|
||||
}
|
||||
} else {
|
||||
// Only fc folds to fc.
|
||||
expected.add_one(fc);
|
||||
}
|
||||
}
|
||||
let mut input = CodePointSet::new();
|
||||
input.add(Interval { first, last });
|
||||
let folded = add_icase_code_points(input);
|
||||
assert_eq!(folded, expected);
|
||||
from = last;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fold_interval() {
|
||||
let locs = [
|
||||
0, 0x894, 0x59ac, 0xfa64, 0x10980, 0x12159, 0x16b8d, 0x1aaa2, 0x1f973, 0x1fcd4,
|
||||
0x20c35, 0x23d8a, 0x276af, 0x2c6b8, 0x2fb25, 0x30b9b, 0x338ad, 0x35ab3, 0x38d37,
|
||||
0x3bfa7, 0x3fba6, 0x404c9, 0x44572, 0x480c9, 0x4b5c4, 0x4f371, 0x5a9fa, 0x5ad6c,
|
||||
0x5e395, 0x5f103, 0x5fa98, 0x617fa, 0x6500e, 0x68890, 0x6a3fc, 0x6eab3, 0x704a6,
|
||||
0x70c22, 0x72efb, 0x737cc, 0x76796, 0x79da8, 0x7a450, 0x7b023, 0x7cc5c, 0x82027,
|
||||
0x84ef4, 0x8ac66, 0x8b898, 0x8bd1a, 0x95841, 0x98a48, 0x9e6cd, 0xa035a, 0xa41fb,
|
||||
0xa50e3, 0xa6387, 0xa7ba1, 0xaad9a, 0xabed8, 0xacc88, 0xb2737, 0xb31b1, 0xb6daf,
|
||||
0xb7ff4, 0xba2b4, 0xbde4f, 0xbe38b, 0xbe7a5, 0xc4eb2, 0xc5670, 0xc7703, 0xc995d,
|
||||
0xccb72, 0xcdfe3, 0xcfc99, 0xd09eb, 0xd2773, 0xd357d, 0xd6696, 0xd9aec, 0xdc3fa,
|
||||
0xdc8ae, 0xdc9d5, 0xde31d, 0xe2edb, 0xe652b, 0xe92d5, 0xebf2d, 0xee335, 0xef45f,
|
||||
0xf4280, 0xf74b1, 0xf9ac4, 0xfafca, 0x10208d, 0x107d63, 0x10821e, 0x108818, 0x10911f,
|
||||
0x10b6fd, 0x10FFFF,
|
||||
];
|
||||
for (idx, &first) in locs.iter().enumerate() {
|
||||
// Keep a running set of the folded code points we expect to be in the
|
||||
// range [first, last].
|
||||
let mut expected = CodePointSet::default();
|
||||
let mut from = first;
|
||||
for &last in &locs[idx..] {
|
||||
// Add characters to expected which do not fold to themselves.
|
||||
for c in from..=last {
|
||||
let fc = fold(c);
|
||||
if fc != c {
|
||||
expected.add_one(fc);
|
||||
}
|
||||
}
|
||||
let mut cps = CodePointSet::default();
|
||||
fold_interval(Interval { first, last }, &mut cps);
|
||||
assert_eq!(cps.intervals(), expected.intervals());
|
||||
|
||||
from = last;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+33975
File diff suppressed because it is too large
Load Diff
Vendored
+286
@@ -0,0 +1,286 @@
|
||||
use crate::bytesearch::ByteBitmap;
|
||||
use crate::codepointset::CODE_POINT_MAX;
|
||||
use crate::codepointset::Interval;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
use core::cmp::Ordering;
|
||||
use core::ops::{Index, IndexMut};
|
||||
use core::slice::SliceIndex;
|
||||
|
||||
// A macro which expresses either checked or unchecked reachability, depending on prohibit-unsafe.
|
||||
macro_rules! rs_unreachable {
|
||||
() => {{
|
||||
if cfg!(feature = "prohibit-unsafe") {
|
||||
unreachable!();
|
||||
} else {
|
||||
unsafe { core::hint::unreachable_unchecked() }
|
||||
}
|
||||
}};
|
||||
($msg:expr) => {
|
||||
if cfg!(feature = "prohibit-unsafe") {
|
||||
unreachable!($msg);
|
||||
} else {
|
||||
unsafe { core::hint::unreachable_unchecked() }
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// A trait which performs bounds checking only in debug mode.
|
||||
pub trait DebugCheckIndex<Idx>: Index<Idx> + IndexMut<Idx> {
|
||||
fn iat(&self, index: Idx) -> &Self::Output;
|
||||
fn mat(&mut self, index: Idx) -> &mut Self::Output;
|
||||
}
|
||||
|
||||
impl<Idx, T> DebugCheckIndex<Idx> for Vec<T>
|
||||
where
|
||||
Idx: SliceIndex<[T]> + Clone,
|
||||
{
|
||||
#[inline(always)]
|
||||
fn iat(&self, idx: Idx) -> &Self::Output {
|
||||
debug_assert!(self.get(idx.clone()).is_some(), "Index out of bounds");
|
||||
if cfg!(feature = "prohibit-unsafe") {
|
||||
self.index(idx)
|
||||
} else {
|
||||
unsafe { self.get_unchecked(idx) }
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn mat(&mut self, idx: Idx) -> &mut Self::Output {
|
||||
debug_assert!(self.get(idx.clone()).is_some(), "Index out of bounds");
|
||||
if cfg!(feature = "prohibit-unsafe") {
|
||||
self.index_mut(idx)
|
||||
} else {
|
||||
unsafe { self.get_unchecked_mut(idx) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<Idx, T> DebugCheckIndex<Idx> for [T]
|
||||
where
|
||||
Idx: SliceIndex<[T]> + Clone,
|
||||
{
|
||||
#[inline(always)]
|
||||
fn iat(&self, idx: Idx) -> &Self::Output {
|
||||
debug_assert!(self.get(idx.clone()).is_some(), "Index out of bounds");
|
||||
if cfg!(feature = "prohibit-unsafe") {
|
||||
self.index(idx)
|
||||
} else {
|
||||
unsafe { self.get_unchecked(idx) }
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn mat(&mut self, idx: Idx) -> &mut Self::Output {
|
||||
debug_assert!(self.get(idx.clone()).is_some(), "Index out of bounds");
|
||||
if cfg!(feature = "prohibit-unsafe") {
|
||||
self.index_mut(idx)
|
||||
} else {
|
||||
unsafe { self.get_unchecked_mut(idx) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function for matching u32s against chars.
|
||||
// It would be pleasant if you could pattern-match u32s against chars, but Rust does not allow this.
|
||||
// Convert a u32 to a char, except if the conversion fails, return the largest char.
|
||||
// Be careful to not use the result of this conversion except to pattern match against literals.
|
||||
pub fn to_char_sat(c: u32) -> char {
|
||||
char::from_u32(c).unwrap_or(core::char::MAX)
|
||||
}
|
||||
|
||||
/// \return the first byte of a UTF-8 encoded code point.
|
||||
/// We do not use char because we don't want to deal with failing on surrogates.
|
||||
#[inline(always)]
|
||||
pub fn utf8_first_byte(cp: u32) -> u8 {
|
||||
debug_assert!(cp <= CODE_POINT_MAX);
|
||||
if cp < 0x80 {
|
||||
// One byte encoding.
|
||||
cp as u8
|
||||
} else if cp < 0x800 {
|
||||
// Two byte encoding.
|
||||
((cp >> 6) & 0x1F) as u8 | 0b1100_0000
|
||||
} else if cp < 0x10000 {
|
||||
// Three byte encoding.
|
||||
((cp >> 12) & 0x0F) as u8 | 0b1110_0000
|
||||
} else {
|
||||
// Four byte encoding.
|
||||
((cp >> 18) & 0x07) as u8 | 0b1111_0000
|
||||
}
|
||||
}
|
||||
|
||||
/// Add all of the first bytes of a code point interval to a byte bitmap.
|
||||
pub fn add_utf8_first_bytes_to_bitmap(interval: Interval, bitmap: &mut ByteBitmap) {
|
||||
// Note this is an inclusive interval.
|
||||
let Interval { first, last } = interval;
|
||||
let ranges = [
|
||||
(first, last.min(0x7F)), // 1 byte range
|
||||
(first.max(0x80), last.min(0x7FF)), // 2 byte range
|
||||
(first.max(0x800), last.min(0xFFFF)), // 3 byte range
|
||||
(first.max(0x10000), last), // 4 byte range
|
||||
];
|
||||
for (first, last) in ranges.into_iter() {
|
||||
if first <= last {
|
||||
for byte in utf8_first_byte(first)..=utf8_first_byte(last) {
|
||||
bitmap.set(byte);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait SliceHelp {
|
||||
type Item;
|
||||
|
||||
/// Given that self is sorted according to f, returns the range of indexes
|
||||
/// where f indicates equal elements.
|
||||
fn equal_range_by<'a, F>(&'a self, f: F) -> core::ops::Range<usize>
|
||||
where
|
||||
F: FnMut(&'a Self::Item) -> Ordering;
|
||||
}
|
||||
|
||||
impl<T> SliceHelp for [T] {
|
||||
type Item = T;
|
||||
fn equal_range_by<'a, F>(&'a self, mut f: F) -> core::ops::Range<usize>
|
||||
where
|
||||
F: FnMut(&'a Self::Item) -> Ordering,
|
||||
{
|
||||
let left = self
|
||||
.binary_search_by(|v| f(v).then(Ordering::Greater))
|
||||
.unwrap_err();
|
||||
let right = self[left..]
|
||||
.binary_search_by(|v| f(v).then(Ordering::Less))
|
||||
.unwrap_err()
|
||||
+ left;
|
||||
left..right
|
||||
}
|
||||
}
|
||||
|
||||
// Given a byte \p b, keep its low \p mask bits, and then shift left by \p shift.
|
||||
const fn mask_shift(b: u8, mask: u8, shift: u8) -> u32 {
|
||||
let masked = b & ((1 << mask) - 1);
|
||||
(masked as u32) << (shift as u32)
|
||||
}
|
||||
|
||||
// Number of significant bits in a utf8 continuation byte.
|
||||
const UTF8_CONT_SIGBITS: u8 = 6;
|
||||
|
||||
/// \return true if \p b is a UTF8 continutation byte.
|
||||
#[inline(always)]
|
||||
pub fn is_utf8_continuation(b: u8) -> bool {
|
||||
(b & 0b1100_0000) == 0b1000_0000
|
||||
}
|
||||
|
||||
// Construct a code point from a list of bytes.
|
||||
#[inline(always)]
|
||||
pub fn utf8_w2(b0: u8, b1: u8) -> u32 {
|
||||
debug_assert!(!is_utf8_continuation(b0) && is_utf8_continuation(b1));
|
||||
debug_assert!(b0 >> 5 == 0b110);
|
||||
mask_shift(b0, 5, UTF8_CONT_SIGBITS) | mask_shift(b1, UTF8_CONT_SIGBITS, 0)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn utf8_w3(b0: u8, b1: u8, b2: u8) -> u32 {
|
||||
debug_assert!(
|
||||
!is_utf8_continuation(b0) && is_utf8_continuation(b1) && is_utf8_continuation(b2)
|
||||
);
|
||||
debug_assert!(b0 >> 4 == 0b1110);
|
||||
mask_shift(b0, 4, 2 * UTF8_CONT_SIGBITS)
|
||||
| mask_shift(b1, UTF8_CONT_SIGBITS, UTF8_CONT_SIGBITS)
|
||||
| mask_shift(b2, UTF8_CONT_SIGBITS, 0)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn utf8_w4(b0: u8, b1: u8, b2: u8, b3: u8) -> u32 {
|
||||
debug_assert!(
|
||||
!is_utf8_continuation(b0)
|
||||
&& is_utf8_continuation(b1)
|
||||
&& is_utf8_continuation(b2)
|
||||
&& is_utf8_continuation(b3)
|
||||
);
|
||||
debug_assert!(b0 >> 3 == 0b11110);
|
||||
mask_shift(b0, 3, 3 * UTF8_CONT_SIGBITS)
|
||||
| mask_shift(b1, UTF8_CONT_SIGBITS, 2 * UTF8_CONT_SIGBITS)
|
||||
| mask_shift(b2, UTF8_CONT_SIGBITS, UTF8_CONT_SIGBITS)
|
||||
| mask_shift(b3, UTF8_CONT_SIGBITS, 0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ByteBitmap, Interval, SliceHelp, add_utf8_first_bytes_to_bitmap, utf8_first_byte};
|
||||
|
||||
#[test]
|
||||
fn ranges() {
|
||||
let vals = [0, 1, 2, 3, 4, 4, 4, 7, 8, 9, 9];
|
||||
let fast_er = |needle: usize| vals.equal_range_by(|v| v.cmp(&needle));
|
||||
let slow_er = |needle: usize| {
|
||||
let mut left = 0;
|
||||
while left < vals.len() && vals[left] < needle {
|
||||
left += 1
|
||||
}
|
||||
let mut right = left;
|
||||
while right < vals.len() && vals[right] == needle {
|
||||
right += 1
|
||||
}
|
||||
left..right
|
||||
};
|
||||
|
||||
for i in 0..10 {
|
||||
assert_eq!(fast_er(i), slow_er(i))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utf8() {
|
||||
for &cp in &[
|
||||
0x0,
|
||||
0x7,
|
||||
0xFF,
|
||||
0x80,
|
||||
0xABC,
|
||||
0x7FF,
|
||||
0x800,
|
||||
0x801,
|
||||
0xFFFF,
|
||||
0x10000,
|
||||
0x10001,
|
||||
0x1FFFF,
|
||||
super::CODE_POINT_MAX - 1,
|
||||
super::CODE_POINT_MAX,
|
||||
] {
|
||||
use super::{utf8_w2, utf8_w3, utf8_w4};
|
||||
let mut buff = [0; 4];
|
||||
let s = core::char::from_u32(cp).unwrap().encode_utf8(&mut buff);
|
||||
let bytes = s.as_bytes();
|
||||
assert_eq!(bytes[0], utf8_first_byte(cp));
|
||||
|
||||
match bytes.len() {
|
||||
1 => assert_eq!(bytes[0] as u32, cp),
|
||||
2 => assert_eq!(utf8_w2(bytes[0], bytes[1]), cp),
|
||||
3 => assert_eq!(utf8_w3(bytes[0], bytes[1], bytes[2]), cp),
|
||||
4 => assert_eq!(utf8_w4(bytes[0], bytes[1], bytes[2], bytes[3]), cp),
|
||||
_ => panic!("Unexpected utf8 sequence length"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_utf8_first_bytes_to_bitmap() {
|
||||
let locs = [
|
||||
0, 1, 2, 3, 17, 25, 34, 99, 127, 128, 150, 255, 256, 257, 511, 512, 513, 0x7FF, 0x800,
|
||||
0x905, 0xA123, 0xFF0F, 0xFFFF, 0x10000, 0x10001, 0x1FFFF, 0x20001, 0x2FFF7, 0x50001,
|
||||
0x100000, 0x10FFFE, 0x10FFFF,
|
||||
];
|
||||
for (idx, &first) in locs.iter().enumerate() {
|
||||
let mut expected = ByteBitmap::default();
|
||||
for &last in locs[idx..].iter() {
|
||||
assert!(first <= last);
|
||||
for cp in first..=last {
|
||||
expected.set(utf8_first_byte(cp));
|
||||
}
|
||||
let mut bitmap = ByteBitmap::default();
|
||||
add_utf8_first_bytes_to_bitmap(Interval { first, last }, &mut bitmap);
|
||||
assert_eq!(bitmap, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user