mirror of
https://github.com/pjh456/OxideJS.git
synced 2026-10-23 18:21:13 +08:00
fix: 正则非 u 标志按码元匹配，u/v 标志按码点
This commit is contained in:
@@ -634,8 +634,10 @@ fn rx_search<H: VmHost>(
|
||||
}
|
||||
};
|
||||
|
||||
// 搜索起点：global/sticky 自 lastIndex 起，其余自 0。
|
||||
// 搜索起点：global/sticky 自 lastIndex 起，其余自 0。u/v 标志下起点须
|
||||
// 落在码点边界（低 surrogate 位钳回代理对起点，u 标志匹配只起于码点边界）。
|
||||
let start = if tracks_last_index { last_index } else { 0 };
|
||||
let start = text.advance_start(regex, start);
|
||||
|
||||
// 越界短路：lastIndex 超出串长必不匹配，规范先 Set 0 再空结果。
|
||||
if start > text.len_units() {
|
||||
@@ -646,7 +648,7 @@ fn rx_search<H: VmHost>(
|
||||
}
|
||||
|
||||
// 匹配范围取臂原生命径（Str 臂字节、Units 臂码元）；sticky 加后置锚定
|
||||
// 过滤——底层引擎对 y 不原生锚定，命中起点须恰在 lastIndex 才算有效。
|
||||
// 过滤——底层引擎对 y 不原生锚定，命中起点须恰在推进后的 lastIndex 才算有效。
|
||||
let match_result = text.find_from_units(regex, start);
|
||||
let anchored = if is_sticky {
|
||||
match_result.filter(|m| text.unit_pos(m.range().start) == start)
|
||||
|
||||
@@ -73,52 +73,135 @@ impl<'a> MatchText<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// 从码元位置开始的首个匹配（Str 臂内部换算为字节位置）。
|
||||
/// 从码元位置开始的首个匹配。
|
||||
///
|
||||
/// # 边界与前提
|
||||
/// - Str 臂仅 ASCII 走字节路径（字节==码元，u 与非-u 均正确）；非 ASCII
|
||||
/// 编码为单元后按正则 flags 分派：u/v 码点语义、其余码元语义。
|
||||
pub(crate) fn find_from_units(&self, regex: ®ress::Regex, start_units: usize) -> Option<regress::Match> {
|
||||
match self {
|
||||
MatchText::Str(s) => regex.find_from(s, unit_to_byte(s, start_units)).next(),
|
||||
MatchText::Units(u) => regex.find_from_utf16(u, start_units).next(),
|
||||
MatchText::Str(s) => {
|
||||
if s.is_ascii() {
|
||||
return regex.find_from(s, start_units).next();
|
||||
}
|
||||
first_match_units(regex, &self.units(), start_units)
|
||||
}
|
||||
MatchText::Units(u) => first_match_units(regex, u, start_units),
|
||||
}
|
||||
}
|
||||
|
||||
/// 按序消费全部非重叠匹配（自码元 0 起）。
|
||||
///
|
||||
/// # 边界与前提
|
||||
/// - Str 臂仅 ASCII 走字节路径（字节==码元，u 与非-u 均正确）；非 ASCII
|
||||
/// 编码为单元后按正则 flags 分派：u/v 码点语义、其余码元语义。
|
||||
pub(crate) fn for_each_match(&self, regex: ®ress::Regex, mut f: impl FnMut(®ress::Match)) {
|
||||
match self {
|
||||
MatchText::Str(s) => {
|
||||
if s.is_ascii() {
|
||||
for m in regex.find_iter(s) {
|
||||
f(&m);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let u = self.units();
|
||||
if regex.flags().unicode || regex.flags().unicode_sets {
|
||||
for m in regex.find_from_utf16(&u, 0) {
|
||||
f(&m);
|
||||
}
|
||||
} else {
|
||||
for m in regex.find_from_ucs2(&u, 0) {
|
||||
f(&m);
|
||||
}
|
||||
}
|
||||
}
|
||||
MatchText::Units(u) => {
|
||||
if regex.flags().unicode || regex.flags().unicode_sets {
|
||||
for m in regex.find_from_utf16(u, 0) {
|
||||
f(&m);
|
||||
}
|
||||
} else {
|
||||
for m in regex.find_from_ucs2(u, 0) {
|
||||
f(&m);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 匹配片段为单元序列（start/end 取臂自身匹配范围口径：Str 臂字节、
|
||||
/// Units 臂码元——臂由文本通道钉死，调用方直接传 `m.range()`）。
|
||||
/// 匹配片段为单元序列（start/end 取臂自身匹配范围口径：Str 臂 ASCII
|
||||
/// 字节==码元、非 ASCII 即码元，Units 臂码元——调用方直接传 `m.range()`）。
|
||||
pub(crate) fn slice(&self, start: usize, end: usize) -> Cow<'a, [u16]> {
|
||||
match self {
|
||||
MatchText::Str(s) => {
|
||||
let sub = &s[start..end];
|
||||
Cow::Owned(if sub.is_ascii() {
|
||||
sub.as_bytes().iter().map(|&b| b as u16).collect()
|
||||
} else {
|
||||
sub.encode_utf16().collect()
|
||||
})
|
||||
if s.is_ascii() {
|
||||
return Cow::Owned(s.as_bytes()[start..end].iter().map(|&b| b as u16).collect());
|
||||
}
|
||||
let u = self.units();
|
||||
Cow::Owned(u[start..end].to_vec())
|
||||
}
|
||||
MatchText::Units(u) => Cow::Owned(u[start..end].to_vec()),
|
||||
}
|
||||
}
|
||||
|
||||
/// 匹配范围端点换算到码元口径（Str 臂范围是字节口径，Units 臂即码元）。
|
||||
/// 匹配范围端点换算到码元口径：两臂匹配范围均已是码元口径
|
||||
/// （Str 臂 ASCII 字节==码元、非 ASCII 走单元入口），恒等返回。
|
||||
pub(crate) fn unit_pos(&self, pos: usize) -> usize {
|
||||
match self {
|
||||
MatchText::Str(s) => byte_to_unit(s, pos),
|
||||
MatchText::Units(_) => pos,
|
||||
pos
|
||||
}
|
||||
|
||||
/// 匹配起点按码点边界钳制：u/v 标志下起点须落在码点边界上（低
|
||||
/// surrogate 位钳回代理对起点，u 标志匹配只起于码点边界），其余标志
|
||||
/// 恒等。ASCII 文本字节==码元，一切位置皆边界。
|
||||
pub(crate) fn advance_start(&self, regex: ®ress::Regex, start: usize) -> usize {
|
||||
if !(regex.flags().unicode || regex.flags().unicode_sets) {
|
||||
return start;
|
||||
}
|
||||
match self {
|
||||
MatchText::Str(s) if s.is_ascii() => start,
|
||||
_ => clamp_code_point_start(&self.units(), start),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 码点起点钳制：低 surrogate 位（前一单元为高 surrogate）非码点起点，
|
||||
/// 钳回代理对起点；其余位置（含串尾）恒等。
|
||||
fn clamp_code_point_start(u: &[u16], idx: usize) -> usize {
|
||||
if idx > 0 && idx < u.len() && is_low_surrogate(u[idx]) && is_high_surrogate(u[idx - 1]) {
|
||||
idx - 1
|
||||
} else {
|
||||
idx
|
||||
}
|
||||
}
|
||||
|
||||
/// 给定码元位置之后的下一个码点边界（空匹配游标推进用）：跳过低 surrogate
|
||||
/// 位到下一码点起点；位置在串尾时返回串长+1（触发耗尽守卫）。
|
||||
pub(crate) fn next_code_point_boundary(u: &[u16], idx: usize) -> usize {
|
||||
let mut j = idx + 1;
|
||||
while j < u.len() && is_low_surrogate(u[j]) && is_high_surrogate(u[j - 1]) {
|
||||
j += 1;
|
||||
}
|
||||
j
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_low_surrogate(u: u16) -> bool {
|
||||
(0xDC00..=0xDFFF).contains(&u)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_high_surrogate(u: u16) -> bool {
|
||||
(0xD800..=0xDBFF).contains(&u)
|
||||
}
|
||||
|
||||
/// 单元序列上的首个匹配：u/v 标志走码点入口（UTF-16 单元按码点分组），
|
||||
/// 其余走码元入口（每单元独立元素）。码点入口的起点须落在码点边界上：
|
||||
/// 低 surrogate 位的起点钳回代理对起点（u 标志匹配只起于码点边界）。
|
||||
pub(crate) fn first_match_units(regex: ®ress::Regex, u: &[u16], start: usize) -> Option<regress::Match> {
|
||||
if regex.flags().unicode || regex.flags().unicode_sets {
|
||||
regex.find_from_utf16(u, clamp_code_point_start(u, start)).next()
|
||||
} else {
|
||||
regex.find_from_ucs2(u, start).next()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -205,30 +288,6 @@ pub(crate) fn is_trim_unit(u: u16) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// 良形文本的字节偏移 → 码元数（Str 臂匹配范围/index 的码元换算入口）。
|
||||
pub(crate) fn byte_to_unit(s: &str, byte: usize) -> usize {
|
||||
if s.is_ascii() {
|
||||
byte
|
||||
} else {
|
||||
s[..byte].encode_utf16().count()
|
||||
}
|
||||
}
|
||||
|
||||
/// 良形文本的码元位置 → 字节偏移（位置须落在字符边界：匹配游标恒对齐）。
|
||||
pub(crate) fn unit_to_byte(s: &str, unit: usize) -> usize {
|
||||
if s.is_ascii() {
|
||||
return unit.min(s.len());
|
||||
}
|
||||
let mut acc = 0usize;
|
||||
for (byte, ch) in s.char_indices() {
|
||||
if unit == acc {
|
||||
return byte;
|
||||
}
|
||||
acc += ch.len_utf16();
|
||||
}
|
||||
s.len()
|
||||
}
|
||||
|
||||
/// 大小写/规范化按"良形段"分段映射：整段经 UTF-8 往返调用映射函数，孤立
|
||||
/// surrogate 单元原样透传（映射对码点定义，对孤立 surrogate 无定义）。
|
||||
pub(crate) fn map_well_formed_segments(units: &[u16], mut f: impl FnMut(&str) -> String) -> Vec<u16> {
|
||||
|
||||
@@ -10,8 +10,8 @@ use crate::regexp::build_groups_object;
|
||||
|
||||
use super::common::{try_string, MatchText};
|
||||
use super::{
|
||||
as_units, byte_to_unit, find_units, make_string_array_values, make_units_array, map_well_formed_segments,
|
||||
split_limit_to_uint32, this_text, this_units, unit_to_byte,
|
||||
as_units, find_units, first_match_units, make_string_array_values, make_units_array, map_well_formed_segments,
|
||||
next_code_point_boundary, split_limit_to_uint32, this_text, this_units,
|
||||
};
|
||||
|
||||
// ── 正则替换（单元口径） ────────────────────────────────────────────────
|
||||
@@ -1050,8 +1050,9 @@ pub fn string_match_all_next<H: VmHost>(vm: &mut H, args: &[u8]) -> NativeResult
|
||||
// d 标志门控 indices 产出（flags 串实例字段直读，与 exec 同口径）。
|
||||
let has_indices = crate::regexp::regexp_has_flag(vm, re_obj, 'd');
|
||||
|
||||
// 按 input 载荷形态分臂：Flat 走字节通道（游标经码元↔字节换算），
|
||||
// 其余走单元通道（游标即码元）。
|
||||
// 按 input 载荷形态分臂：ASCII Flat 走字节通道（字节==码元，游标与匹配
|
||||
// 范围同口径，无需换算）；非 ASCII Flat 与其余载荷走单元通道，按标志
|
||||
// 分派（u/v 码点、其余码元，见 first_match_units）。
|
||||
// SAFETY: input_val 已校验为字符串值，裸指针借用压缩到单个表达式。
|
||||
let sp = unsafe { &*input_val.as_string_ptr() };
|
||||
// 耗尽守卫：空匹配正则（如 /(?:)/g、/a*/g）在串尾会反复命中同一末端空
|
||||
@@ -1061,9 +1062,11 @@ pub fn string_match_all_next<H: VmHost>(vm: &mut H, args: &[u8]) -> NativeResult
|
||||
return make_match_done_result(vm, JsValue::undefined());
|
||||
}
|
||||
// 元素按值混排：匹配片段物化字符串值，未匹配捕获组为 undefined。
|
||||
let (match_start, next_idx, parts, m_opt): (usize, usize, Vec<JsValue>, Option<regress::Match>) = if sp.is_flat() {
|
||||
let (match_start, next_idx, parts, m_opt): (usize, usize, Vec<JsValue>, Option<regress::Match>) = if sp.is_flat()
|
||||
&& sp.as_str().is_ascii()
|
||||
{
|
||||
let s = sp.as_str();
|
||||
match regex.find_from(s, unit_to_byte(s, idx)).next() {
|
||||
match regex.find_from(s, idx).next() {
|
||||
Some(m) => {
|
||||
let range = m.range();
|
||||
let mut parts = Vec::with_capacity(m.captures.len() + 1);
|
||||
@@ -1076,16 +1079,15 @@ pub fn string_match_all_next<H: VmHost>(vm: &mut H, args: &[u8]) -> NativeResult
|
||||
}
|
||||
// 空匹配（range 无推进）须推进至少一个码元，否则同一位置反复
|
||||
// 空匹配死循环；非空匹配游标落匹配末尾（码元口径）。
|
||||
let start_units = byte_to_unit(s, range.start);
|
||||
let end_units = byte_to_unit(s, range.end);
|
||||
let next_idx = if range.end > range.start { end_units } else { end_units + 1 };
|
||||
(start_units, next_idx, parts, Some(m))
|
||||
let next_idx = if range.end > range.start { range.end } else { range.end + 1 };
|
||||
(range.start, next_idx, parts, Some(m))
|
||||
}
|
||||
None => (0, 0, Vec::new(), None),
|
||||
}
|
||||
} else {
|
||||
let u = sp.units();
|
||||
match regex.find_from_utf16(&u, idx).next() {
|
||||
let is_code_point = regex.flags().unicode || regex.flags().unicode_sets;
|
||||
match first_match_units(regex, &u, idx) {
|
||||
Some(m) => {
|
||||
let range = m.range();
|
||||
let mut parts = Vec::with_capacity(m.captures.len() + 1);
|
||||
@@ -1096,7 +1098,15 @@ pub fn string_match_all_next<H: VmHost>(vm: &mut H, args: &[u8]) -> NativeResult
|
||||
None => parts.push(JsValue::undefined()),
|
||||
}
|
||||
}
|
||||
let next_idx = if range.end > range.start { range.end } else { range.end + 1 };
|
||||
// 非空匹配游标落匹配末尾；空匹配推进到下一码点边界（u/v
|
||||
// 标志）或下一码元（其余），串尾空匹配推进到串长+1 触发耗尽。
|
||||
let next_idx = if range.end > range.start {
|
||||
range.end
|
||||
} else if is_code_point {
|
||||
next_code_point_boundary(&u, range.end)
|
||||
} else {
|
||||
range.end + 1
|
||||
};
|
||||
(range.start, next_idx, parts, Some(m))
|
||||
}
|
||||
None => (0, 0, Vec::new(), None),
|
||||
|
||||
Vendored
+8
-5
@@ -77,16 +77,19 @@ upstream 0.11.1, other than this note; upstream repo-only files —
|
||||
`utf16` feature) to a per-compilation runtime flag, so enabling `utf16`
|
||||
no longer disables the byte passes on the str path:
|
||||
- `src/optimizer.rs` — the literal-byte pass is gated by the new
|
||||
`optimize_with_byte_literals(ire, byte_literals)` entry; `optimize`
|
||||
keeps the previous behavior as its wrapper.
|
||||
`optimize_with_byte_literals(ire, byte_literals, decompose_nonbmp)`
|
||||
entry; `optimize` keeps the previous behavior as its wrapper. The
|
||||
`decompose_nonbmp` flag splits non-BMP codepoint literals into
|
||||
surrogate pairs for the code-unit (UCS-2) unit IR only.
|
||||
- `src/classicalbacktrack.rs` — the start-predicate dispatch in the
|
||||
matcher's next-match loop switches on input encoding at runtime: byte
|
||||
inputs (UTF-8/ASCII) keep the anchored/byte-positioning fast paths,
|
||||
unit inputs (UTF-16/UCS-2) fall back to position-by-position tries.
|
||||
- `src/api.rs` — `Regex` retains the pattern's source code-point
|
||||
sequence plus a lazily materialized unit-side IR (`OnceLock`),
|
||||
re-parsed with the byte-literal pass disabled on first unit-input
|
||||
entry; the unit entry points compile through that IR.
|
||||
sequence plus lazily materialized unit-side IRs (two `OnceLock`s:
|
||||
code-point-oriented and code-unit-oriented), re-parsed with the
|
||||
byte-literal pass disabled on first unit-input entry; the unit entry
|
||||
points compile through those IRs.
|
||||
3. `Cargo.toml` — the upstream workspace-members table (for the repo's
|
||||
`regress-tool`/`gen-unicode` subdirectories, not vendored here) is
|
||||
replaced by a standalone empty `[workspace]` table, and the `utf16`
|
||||
|
||||
Vendored
+34
-9
@@ -381,8 +381,12 @@ pub struct Regex {
|
||||
pattern: Option<Vec<u32>>,
|
||||
|
||||
// 单元向编译产物（未跑字节字面量 pass），首次单元输入入口时惰性物化。
|
||||
// 两路缓存：码点向入口（`find_from_utf16`）不拆非 BMP 码点字面量；
|
||||
// 码元向入口（`find_from_ucs2`）拆为 surrogate 对。两 IR 不得共用。
|
||||
#[cfg_attr(not(feature = "utf16"), allow(dead_code))]
|
||||
units_cr: OnceLock<CompiledRegex>,
|
||||
#[cfg_attr(not(feature = "utf16"), allow(dead_code))]
|
||||
units_cr_ucs2: OnceLock<CompiledRegex>,
|
||||
}
|
||||
|
||||
impl From<CompiledRegex> for Regex {
|
||||
@@ -391,6 +395,7 @@ impl From<CompiledRegex> for Regex {
|
||||
cr,
|
||||
pattern: None,
|
||||
units_cr: OnceLock::new(),
|
||||
units_cr_ucs2: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -441,15 +446,36 @@ impl Regex {
|
||||
cr,
|
||||
pattern: Some(pattern),
|
||||
units_cr: OnceLock::new(),
|
||||
units_cr_ucs2: OnceLock::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// 单元向编译产物：首次使用时按同源码同 flags 重解析、关字节字面量 pass
|
||||
/// 重新优化并 emit，得到可在 UTF-16/UCS-2 单元输入上执行的 IR。
|
||||
/// 编译本正则所用的 flags（u/v 标志决定码点语义，供调用方选择
|
||||
/// 码点向或码元向的单元输入入口）。
|
||||
pub fn flags(&self) -> &Flags {
|
||||
&self.cr.flags
|
||||
}
|
||||
|
||||
/// 单元向编译产物（码点向）：首次使用时按同源码同 flags 重解析、关字节
|
||||
/// 字面量 pass 重新优化并 emit，得到可在 UTF-16 单元输入上执行的 IR。
|
||||
/// 非 BMP 码点字面量保持单码点元素（码点向输入把代理对视作单码点）。
|
||||
/// 无源码构造（`From<CompiledRegex>`）无重解析来源，回退复用既有产物。
|
||||
#[cfg(feature = "utf16")]
|
||||
fn units_cr(&self) -> &CompiledRegex {
|
||||
self.units_cr.get_or_init(|| {
|
||||
self.units_cr.get_or_init(|| self.compile_units(false))
|
||||
}
|
||||
|
||||
/// 单元向编译产物（码元向）：同 `units_cr`，但非 BMP 码点字面量拆为
|
||||
/// 高+低 surrogate 码元，使码元向输入（UCS-2，每单元独立元素）可命中。
|
||||
#[cfg(feature = "utf16")]
|
||||
fn units_cr_ucs2(&self) -> &CompiledRegex {
|
||||
self.units_cr_ucs2.get_or_init(|| self.compile_units(true))
|
||||
}
|
||||
|
||||
/// 两路单元向缓存共用的惰性编译体：`decompose` 决定非 BMP 码点字面量
|
||||
/// 是否拆为 surrogate 对。
|
||||
#[cfg(feature = "utf16")]
|
||||
fn compile_units(&self, decompose: bool) -> CompiledRegex {
|
||||
let Some(pattern) = &self.pattern else {
|
||||
return self.cr.clone();
|
||||
};
|
||||
@@ -458,10 +484,9 @@ impl Regex {
|
||||
let mut ire =
|
||||
parse::try_parse(pattern.iter().copied(), self.cr.flags).expect("构造期已成功解析的模式重解析失败");
|
||||
if !self.cr.flags.no_opt {
|
||||
optimizer::optimize_with_byte_literals(&mut ire, false);
|
||||
optimizer::optimize_with_byte_literals(&mut ire, false, decompose);
|
||||
}
|
||||
emit::emit(&ire)
|
||||
})
|
||||
}
|
||||
|
||||
/// Searches `text` to find the first match.
|
||||
@@ -528,7 +553,7 @@ impl Regex {
|
||||
start: usize,
|
||||
) -> exec::Matches<super::classicalbacktrack::BacktrackExecutor<'r, indexing::Utf16Input<'t>>>
|
||||
{
|
||||
// 单元向 IR 无字节指令，UTF-16 单元输入可执行。
|
||||
// 码点向 IR 无字节指令，UTF-16 单元输入可执行；代理对按单码点匹配。
|
||||
let cr = self.units_cr();
|
||||
let input = Utf16Input::new(text, cr.flags.unicode);
|
||||
exec::Matches::new(
|
||||
@@ -548,9 +573,9 @@ impl Regex {
|
||||
start: usize,
|
||||
) -> exec::Matches<super::classicalbacktrack::BacktrackExecutor<'r, indexing::Ucs2Input<'t>>>
|
||||
{
|
||||
// 与 `find_from_utf16` 共用单元向 IR：UCS-2 同为单元输入，
|
||||
// str IR 的字节指令在单元索引器上不可执行。
|
||||
let cr = self.units_cr();
|
||||
// 码元向 IR：UCS-2 同为单元输入（str IR 的字节指令在单元索引器上
|
||||
// 不可执行），且非 BMP 码点字面量须拆为 surrogate 对以命中码元。
|
||||
let cr = self.units_cr_ucs2();
|
||||
let input = Ucs2Input::new(text, cr.flags.unicode);
|
||||
exec::Matches::new(
|
||||
super::classicalbacktrack::BacktrackExecutor::new(
|
||||
|
||||
Vendored
+72
-4
@@ -521,17 +521,80 @@ fn simplify_brackets(n: &mut Node, _walk: &Walk) -> PassAction {
|
||||
}
|
||||
}
|
||||
|
||||
/// 非 BMP 码点 → (高 surrogate, 低 surrogate)。
|
||||
fn surrogate_pair(c: u32) -> (u32, u32) {
|
||||
(0xD800 + ((c - 0x10000) >> 10), 0xDC00 + ((c - 0x10000) & 0x3FF))
|
||||
}
|
||||
|
||||
/// 非 BMP 码点字面量拆为高+低 surrogate 对（仅码元向 IR 需要）。
|
||||
///
|
||||
/// 字节指令无法在单元输入（UTF-16/UCS-2）上执行，单元向 IR 的字符字面量是
|
||||
/// 单码元元素：码元向输入（UCS-2）中每个 u16 是独立元素，非 BMP 码点（> 0xFFFF）
|
||||
/// 须拆成连续两个 surrogate 码元才能命中输入中的代理对。码点向输入（UTF-16）
|
||||
/// 把代理对视作单码点元素，不得拆分。
|
||||
fn decompose_nonbmp_chars(n: &mut Node, _w: &Walk) -> PassAction {
|
||||
match n {
|
||||
Node::Char { c, icase } if *c > 0xFFFF => {
|
||||
let (high, low) = surrogate_pair(*c);
|
||||
PassAction::Replace(Node::Cat(vec![
|
||||
Node::Char { c: high, icase: *icase },
|
||||
Node::Char { c: low, icase: *icase },
|
||||
]))
|
||||
}
|
||||
Node::CharSet(chars) if chars.iter().any(|&c| c > 0xFFFF) => {
|
||||
// BMP 码点保持单码元集合，非 BMP 码点各拆为高+低 surrogate 序列，
|
||||
// 两侧以 Alt 合并（单元素集合直接降为 Char）。
|
||||
let bmp: Vec<u32> = chars.iter().copied().filter(|&c| c <= 0xFFFF).collect();
|
||||
let nonbmp: Vec<u32> = chars.iter().copied().filter(|&c| c > 0xFFFF).collect();
|
||||
let bmp_node = match bmp.len() {
|
||||
0 => None,
|
||||
1 => Some(Node::Char { c: bmp[0], icase: false }),
|
||||
_ => Some(Node::CharSet(bmp)),
|
||||
};
|
||||
let mut seqs: Vec<Node> = nonbmp
|
||||
.iter()
|
||||
.map(|&c| {
|
||||
let (high, low) = surrogate_pair(c);
|
||||
Node::Cat(vec![
|
||||
Node::Char { c: high, icase: false },
|
||||
Node::Char { c: low, icase: false },
|
||||
])
|
||||
})
|
||||
.collect();
|
||||
let nonbmp_node = if seqs.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let mut acc = seqs.pop().unwrap();
|
||||
for s in seqs.into_iter().rev() {
|
||||
acc = Node::Alt(Box::new(s), Box::new(acc));
|
||||
}
|
||||
Some(acc)
|
||||
};
|
||||
match (bmp_node, nonbmp_node) {
|
||||
(Some(b), Some(nb)) => PassAction::Replace(Node::Alt(Box::new(b), Box::new(nb))),
|
||||
(b, nb) => PassAction::Replace(b.or(nb).unwrap()),
|
||||
}
|
||||
}
|
||||
_ => PassAction::Keep,
|
||||
}
|
||||
}
|
||||
|
||||
/// 编译主入口：字节字面量 pass 恒开，编译产物与旧的全局 feature 门控形态逐位一致。
|
||||
pub fn optimize(r: &mut Regex) {
|
||||
optimize_with_byte_literals(r, true)
|
||||
optimize_with_byte_literals(r, true, false)
|
||||
}
|
||||
|
||||
/// 与 `optimize` 同一条定点 pass 链，仅字节字面量 pass
|
||||
/// （Char → ByteSequence、ASCII 集合 → ByteSet）由 `byte_literals` 控制。
|
||||
/// （Char → ByteSequence、ASCII 集合 → ByteSet）由 `byte_literals` 控制，
|
||||
/// 非 BMP 码点字面量拆分由 `decompose_nonbmp` 控制。
|
||||
///
|
||||
/// 字节指令无法在单元输入（UTF-16/UCS-2）上执行：面向单元输入的 IR 必须以
|
||||
/// `false` 编译，字节输入（UTF-8）IR 以 `true` 编译保持编译与执行策略不变。
|
||||
pub fn optimize_with_byte_literals(r: &mut Regex, byte_literals: bool) {
|
||||
/// `byte_literals = false` 编译，字节输入（UTF-8）IR 以 `true` 编译保持编译
|
||||
/// 与执行策略不变。非 BMP 码点字面量拆分仅码元向 IR 需要（`decompose_nonbmp`
|
||||
/// 传 `true`）：码元输入中每个 u16 是独立元素，非 BMP 码点须拆成连续两个
|
||||
/// surrogate 码元才能命中；码点向输入（UTF-16）把代理对视作单码点元素，
|
||||
/// 不得拆分。
|
||||
pub fn optimize_with_byte_literals(r: &mut Regex, byte_literals: bool, decompose_nonbmp: bool) {
|
||||
run_pass(r, &mut simplify_brackets);
|
||||
loop {
|
||||
let mut changed = false;
|
||||
@@ -539,6 +602,11 @@ pub fn optimize_with_byte_literals(r: &mut Regex, byte_literals: bool) {
|
||||
if r.flags.icase {
|
||||
changed |= run_pass(r, &mut unfold_icase_chars);
|
||||
}
|
||||
// 码元向 IR：非 BMP 码点字面量拆为 surrogate 对，须先于 unroll/promote
|
||||
// （否则非 BMP 字符会被提升进 1 字符循环而逃过拆分）。
|
||||
if !byte_literals && decompose_nonbmp {
|
||||
changed |= run_pass(r, &mut decompose_nonbmp_chars);
|
||||
}
|
||||
changed |= run_pass(r, &mut unroll_loops);
|
||||
changed |= run_pass(r, &mut promote_1char_loops);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user